[ARVADOS] created: 07a7c1ca8ea3fb014a7eb4922ef762af7da5a0b5

[git at public.curoverse.com]

        at  07a7c1ca8ea3fb014a7eb4922ef762af7da5a0b5 (commit)


commit 07a7c1ca8ea3fb014a7eb4922ef762af7da5a0b5
Author: Tom Clegg <tom at curoverse.com>
Date:   Sun Jan 19 23:55:55 2014 -0800

    Remove +K@{name} hints from collection uuid, link head/tail, job
    output/log, and job_task output.
    
    refs #1881

diff --git a/services/api/db/migrate/20140117231056_normalize_collection_uuid.rb b/services/api/db/migrate/20140117231056_normalize_collection_uuid.rb
new file mode 100644
index 0000000..bec7ec7
--- /dev/null
+++ b/services/api/db/migrate/20140117231056_normalize_collection_uuid.rb
@@ -0,0 +1,91 @@
+class NormalizeCollectionUuid < ActiveRecord::Migration
+  def count_orphans
+    %w(head tail).each do |ht|
+      results = ActiveRecord::Base.connection.execute(<<-EOS)
+SELECT COUNT(links.*)
+ FROM links
+ LEFT JOIN collections c
+   ON links.#{ht}_uuid = c.uuid
+ WHERE (#{ht}_kind='arvados#collection' or #{ht}_uuid ~ '^[0-9a-f]{32,}')
+   AND #{ht}_uuid IS NOT NULL
+   AND #{ht}_uuid NOT IN (SELECT uuid FROM collections)
+EOS
+      puts "#{results.first['count'].to_i} links with #{ht}_uuid pointing nowhere."
+    end
+  end
+
+  def up
+    # Normalize uuids in the collections table to
+    # {hash}+{size}. Existing uuids might be {hash},
+    # {hash}+{size}+K@{instance-name}, {hash}+K@{instance-name}, etc.
+
+    count_orphans
+    puts "Normalizing collection UUIDs."
+
+    update_sql <<-EOS
+UPDATE collections
+ SET uuid = regexp_replace(uuid,'\\+.*','') || '+' || length(manifest_text)
+ WHERE uuid !~ '^[0-9a-f]{32,}\\+[0-9]+'
+   AND (regexp_replace(uuid,'\\+.*','') || '+' || length(manifest_text))
+     NOT IN (SELECT uuid FROM collections)
+EOS
+
+    count_orphans
+    puts "Updating links by stripping +K at .* from *_uuid attributes."
+
+    update_sql <<-EOS
+UPDATE links
+ SET head_uuid = regexp_replace(head_uuid,'\\+K at .*','')
+ WHERE head_uuid like '%+K@%'
+EOS
+    update_sql <<-EOS
+UPDATE links
+ SET tail_uuid = regexp_replace(tail_uuid,'\\+K at .*','')
+ WHERE tail_uuid like '%+K@%'
+EOS
+
+    count_orphans
+    puts "Updating links by searching bare collection hashes using regexp."
+
+    # Next, update {hash} (and any other non-normalized forms) to
+    # {hash}+{size}. This can only work where the corresponding
+    # collection is found in the collections table (otherwise we can't
+    # know the size).
+    %w(head tail).each do |ht|
+      update_sql <<-EOS
+UPDATE links
+ SET #{ht}_uuid = c.uuid
+ FROM collections c
+ WHERE #{ht}_uuid IS NOT NULL
+   AND (#{ht}_kind='arvados#collection' or #{ht}_uuid ~ '^[0-9a-f]{32,}')
+   AND #{ht}_uuid NOT IN (SELECT uuid FROM collections)
+   AND regexp_replace(#{ht}_uuid,'\\+.*','') = regexp_replace(c.uuid,'\\+.*','')
+   AND c.uuid ~ '^[0-9a-f]{32,}\\+[0-9]+'
+EOS
+    end
+
+    count_orphans
+    puts "Stripping \"+K at .*\" from jobs.output, jobs.log, job_tasks.output."
+
+    update_sql <<-EOS
+UPDATE jobs
+ SET output = regexp_replace(output,'\\+K at .*','')
+ WHERE output ~ '^[0-9a-f]{32,}\\+[0-9]+\\+K@\\w+$'
+EOS
+    update_sql <<-EOS
+UPDATE jobs
+ SET log = regexp_replace(log,'\\+K at .*','')
+ WHERE log ~ '^[0-9a-f]{32,}\\+[0-9]+\\+K@\\w+$'
+EOS
+    update_sql <<-EOS
+UPDATE job_tasks
+ SET output = regexp_replace(output,'\\+K at .*','')
+ WHERE output ~ '^[0-9a-f]{32,}\\+[0-9]+\\+K@\\w+$'
+EOS
+
+    puts "Done."
+  end
+
+  def down
+  end
+end

commit a151147d22130c4d9e0f89989ce43db2cec325ed
Author: Tom Clegg <tom at curoverse.com>
Date:   Sun Jan 19 23:34:20 2014 -0800

    Normalize collection UUID to {hash}+{size} before save and lookup.
    
    refs #1881

diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 4b63747..05da5fe 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -49,4 +49,25 @@ class Arvados::V1::CollectionsController < ApplicationController
     end
     show
   end
+
+  protected
+
+  def find_object_by_uuid
+    super
+    if !@object and !params[:uuid].match(/^[0-9a-f]+\+\d+$/)
+      # Normalize the given uuid and search again.
+      hash_part = params[:uuid].match(/^([0-9a-f]*)/)[1]
+      collection = Collection.where('uuid like ?', hash_part + '+%').first
+      if collection
+        # We know the collection exists, and what its real uuid is in
+        # the database. Now, throw out @objects and repeat the usual
+        # lookup procedure. (Returning the collection at this point
+        # would bypass permission checks.)
+        @objects = nil
+        @where = { uuid: collection.uuid }
+        find_objects_for_index
+        @object = @objects.first
+      end
+    end
+  end
 end
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 863e2cb..03e5e4e 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -28,7 +28,9 @@ class Collection < ArvadosModel
     if self.manifest_text.nil? and self.uuid.nil?
       super
     elsif self.manifest_text and self.uuid
-      if self.uuid.gsub(/\+[^,]+/,'') == Digest::MD5.hexdigest(self.manifest_text)
+      self.uuid.gsub! /\+.*/, ''
+      if self.uuid == Digest::MD5.hexdigest(self.manifest_text)
+        self.uuid.gsub! /$/, '+' + self.manifest_text.length.to_s
         true
       else
         errors.add :uuid, 'uuid does not match checksum of manifest_text'

-----------------------------------------------------------------------


hooks/post-receive
--