[ARVADOS] updated: b6283bc4687cced974d5c18bcbd5944bb295da4d

[git at public.curoverse.com]

Summary of changes:
 apps/workbench/app/models/arvados_api_client.rb |  2 +-
 sdk/cli/bin/crunch-job                          | 86 ++++++++++++++++---------
 services/api/app/models/collection.rb           | 54 +++++++++++-----
 3 files changed, 94 insertions(+), 48 deletions(-)

       via  b6283bc4687cced974d5c18bcbd5944bb295da4d (commit)
       via  49f6fea2230872b4f1bad3371af8957c2d661cf0 (commit)
       via  c4c8977ef25cc6805f2cca1dedfc83faecc0bc23 (commit)
       via  de1e5fd5605aaf11b96ef411201e11ac767fe8ba (commit)
       via  d9ab8c81c11120c32864858d7caafe908c408ad5 (commit)
       via  1da9a2a61d66601ab9a02bff439d610ee19c5932 (commit)
      from  622a60e878900b94d71555d72bced70660f065bf (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit b6283bc4687cced974d5c18bcbd5944bb295da4d
Merge: 49f6fea c4c8977
Author: radhika <radhika at curoverse.com>
Date:   Wed Jun 3 15:11:20 2015 -0400

    Merge branch 'master' into 6203-collection-perf-api


commit 49f6fea2230872b4f1bad3371af8957c2d661cf0
Author: radhika <radhika at curoverse.com>
Date:   Wed Jun 3 15:08:56 2015 -0400

    6203: Do not use Keep::Locator.parse to parse locator in some of the most expensive paths.

diff --git a/apps/workbench/app/models/arvados_api_client.rb b/apps/workbench/app/models/arvados_api_client.rb
index ca09aa7..4d549d1 100644
--- a/apps/workbench/app/models/arvados_api_client.rb
+++ b/apps/workbench/app/models/arvados_api_client.rb
@@ -121,7 +121,7 @@ class ArvadosApiClient
         elsif v == false
           query[k] = 0
         else
-          query[k] = JSON.dump(v)
+          query[k] = Oj.dump(v, mode: :compat)
         end
       end
     else
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index f7a715a..7423b25 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -30,6 +30,8 @@ class Collection < ArvadosModel
     t.add :replication_confirmed_at
   end
 
+  LOCATOR_REGEXP = /^([[:xdigit:]]{32})(\+([[:digit:]]+))?(\+([[:upper:]][[:alnum:]+ at _-]*))?$/
+
   def self.attributes_required_columns
     super.merge(
                 # If we don't list manifest_text explicitly, the
@@ -94,13 +96,23 @@ class Collection < ArvadosModel
   def strip_manifest_text
     if self.manifest_text_changed?
       # Remove any permission signatures from the manifest.
-      self.class.munge_manifest_locators!(self[:manifest_text]) do |loc|
-        loc.without_signature.to_s
+      self.class.munge_manifest_locators!(self[:manifest_text]) do |match|
+        self.class.locator_without_signature(match)
       end
     end
     true
   end
 
+  def self.locator_without_signature match
+    without_signature = match[1]
+    without_signature += match[2] if match[2]
+    if match[4]
+      hints = match[4].split('+').reject { |hint| hint.start_with?("A") }
+      without_signature += hints.join('+')
+    end
+    without_signature
+  end
+
   def set_portable_data_hash
     if (portable_data_hash.nil? or
         portable_data_hash == "" or
@@ -198,8 +210,8 @@ class Collection < ArvadosModel
       expire: db_current_time.to_i + Rails.configuration.blob_signature_ttl,
     }
     m = manifest.dup
-    munge_manifest_locators!(m) do |loc|
-      Blob.sign_locator(loc.to_s, signing_opts)
+    munge_manifest_locators!(m) do |match|
+      Blob.sign_locator(locator_without_signature(match), signing_opts)
     end
     return m
   end
@@ -207,13 +219,21 @@ class Collection < ArvadosModel
   def self.munge_manifest_locators! manifest
     # Given a manifest text and a block, yield each locator,
     # and replace it with whatever the block returns.
-    manifest.andand.gsub!(/ [[:xdigit:]]{32}(\+\S+)?/) do |word|
-      if loc = Keep::Locator.parse(word.strip)
-        " " + yield(loc)
-      else
-        " " + word
+    new_lines = []
+    lines = manifest.andand.split("\n")
+    lines.andand.each do |line|
+      words = line.split(' ')
+      new_words = []
+      words.each do |word|
+        if match = LOCATOR_REGEXP.match(word.strip)
+          new_words << yield(match)
+        else
+          new_words << word
+        end
       end
+      new_lines << new_words.join(' ')
     end
+    manifest = new_lines.join("\n")+"\n"
   end
 
   def self.each_manifest_locator manifest
@@ -309,21 +329,23 @@ class Collection < ArvadosModel
   protected
   def portable_manifest_text
     portable_manifest = self[:manifest_text].dup
-    self.class.munge_manifest_locators!(portable_manifest) do |loc|
-      if loc.size
-        loc.hash + '+' + loc.size.to_s
+    portable_manifest = self.class.munge_manifest_locators!(portable_manifest) do |match|
+      if match[2] # size
+        match[1] + match[2]
       else
-        loc.hash
+        match[1]
       end
     end
     portable_manifest
   end
 
   def compute_pdh
+    return @computed_pdh if @computed_pdh
     portable_manifest = portable_manifest_text
-    (Digest::MD5.hexdigest(portable_manifest) +
-     '+' +
-     portable_manifest.bytesize.to_s)
+    @computed_pdh = (Digest::MD5.hexdigest(portable_manifest) +
+                     '+' +
+                     portable_manifest.bytesize.to_s)
+    @computed_pdh
   end
 
   def maybe_clear_replication_confirmed

-----------------------------------------------------------------------


hooks/post-receive
--