[ARVADOS] updated: d0b0a2baec96a678284e3004b13dce85f36ba190

[git at public.curoverse.com]

Summary of changes:
 sdk/python/arvados/commands/copy.py | 46 +++++++++++++++++++++++--------------
 sdk/python/arvados/keep.py          |  5 +++-
 2 files changed, 33 insertions(+), 18 deletions(-)

       via  d0b0a2baec96a678284e3004b13dce85f36ba190 (commit)
      from  edf8405b6a3bbce4a7e11118bb9c706d36856460 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit d0b0a2baec96a678284e3004b13dce85f36ba190
Author: Tim Pierce <twp at curoverse.com>
Date:   Wed Oct 8 17:45:24 2014 -0400

    3699: bug fix: use destination permission hints
    
    Fix copy_collection permission bug: when copying collections, the
    manifest in the collection created at the destination must have
    permission hints returned by the destination Keep server.
    
    Also fixed bug in arvados.keep.KeepClient (use the api_client's API
    token when one is specified)

diff --git a/sdk/python/arvados/commands/copy.py b/sdk/python/arvados/commands/copy.py
index df3d299..c8d3818 100755
--- a/sdk/python/arvados/commands/copy.py
+++ b/sdk/python/arvados/commands/copy.py
@@ -363,33 +363,45 @@ def copy_collection(obj_uuid, src, dst, force=False):
     # Fetch the collection's manifest.
     manifest = c['manifest_text']
 
-    # Enumerate the block locators found in the manifest.
-    collection_blocks = set()
-    src_keep = arvados.keep.KeepClient(api_client=src, num_retries=2)
-    for line in manifest.splitlines():
-        for block_hash in line.split()[1:]:
-            if arvados.util.portable_data_hash_pattern.match(block_hash):
-                collection_blocks.add(block_hash)
-            else:
-                break
-
     # Copy each block from src_keep to dst_keep.
+    # Use the newly signed locators returned from dst_keep to build
+    # a new manifest as we go.
+    src_keep = arvados.keep.KeepClient(api_client=src, num_retries=2)
     dst_keep = arvados.keep.KeepClient(api_client=dst, num_retries=2)
-    for locator in collection_blocks:
-        parts = locator.split('+')
-        logger.info("Copying block %s (%s bytes)", locator, parts[1])
-        data = src_keep.get(locator)
-        dst_keep.put(data)
+
+    dst_manifest = ""
+    dst_locators = {}
+    for line in manifest.splitlines():
+        words = line.split()
+        dst_manifest_line = words[0]
+        for word in words[1:]:
+            try:
+                loc = arvados.KeepLocator(word)
+                blockhash = loc.md5sum
+                # copy this block if we haven't seen it before
+                # (otherwise, just reuse the existing dst_locator)
+                if blockhash not in dst_locators:
+                    logger.info("Copying block %s (%s bytes)", blockhash, loc.size)
+                    data = src_keep.get(word)
+                    dst_locator = dst_keep.put(data)
+                    dst_locators[blockhash] = dst_locator
+                dst_manifest_line += ' ' + dst_locators[blockhash]
+            except ValueError:
+                # If 'word' can't be parsed as a locator,
+                # presume it's a filename.
+                dst_manifest_line += ' ' + word
+        dst_manifest += dst_manifest_line + "\n"
 
     # Copy the manifest and save the collection.
-    logger.debug('saving {} manifest: {}'.format(obj_uuid, manifest))
-    dst_keep.put(manifest)
+    logger.debug('saving {} manifest: {}'.format(obj_uuid, dst_manifest))
+    dst_keep.put(dst_manifest)
 
     if 'uuid' in c:
         del c['uuid']
     if 'owner_uuid' in c:
         del c['owner_uuid']
     c['ensure_unique_name'] = True
+    c['manifest_text'] = dst_manifest
     return dst.collections().create(body=c).execute()
 
 # copy_git_repo(src_git_repo, src, dst, dst_git_repo)
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 37b1c17..0144a10 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -425,7 +425,10 @@ class KeepClient(object):
         if proxy is None:
             proxy = config.get('ARVADOS_KEEP_PROXY')
         if api_token is None:
-            api_token = config.get('ARVADOS_API_TOKEN')
+            if api_client is None:
+                api_token = config.get('ARVADOS_API_TOKEN')
+            else:
+                api_token = api_client.api_token
         elif api_client is not None:
             raise ValueError(
                 "can't build KeepClient with both API client and token")

-----------------------------------------------------------------------


hooks/post-receive
--