[ARVADOS] updated: 313415e33ca374898a371b9f22fc6546793f4688
Git user
git at public.curoverse.com
Mon Apr 3 17:17:05 EDT 2017
Summary of changes:
sdk/cli/test/test_arv-keep-get.rb | 4 +--
sdk/python/arvados/commands/get.py | 65 ++++++++++++++++++++++++--------------
sdk/python/arvados/commands/ls.py | 26 ++++++++++++---
3 files changed, 66 insertions(+), 29 deletions(-)
via 313415e33ca374898a371b9f22fc6546793f4688 (commit)
via 2c0c6e6ba25ec56bab8c224865a7979af577adaa (commit)
via 988e9052575bc93db2909f3b9ef576f1043eddcb (commit)
via 3cc77494ac02e0f64a814cd4a898662fb7b329af (commit)
from c820bfc91be7635739bad0857ba3a385d1334b6a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 313415e33ca374898a371b9f22fc6546793f4688
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Mon Apr 3 18:16:08 2017 -0300
7824: Now a subdirectory can be asked to be listed. Also, enhanced error message when asking for a non existant collection.
diff --git a/sdk/python/arvados/commands/ls.py b/sdk/python/arvados/commands/ls.py
index 918ce5e..c6ca085 100755
--- a/sdk/python/arvados/commands/ls.py
+++ b/sdk/python/arvados/commands/ls.py
@@ -5,6 +5,7 @@ from __future__ import print_function
import argparse
import collections
import logging
+import re
import sys
import arvados
@@ -20,7 +21,7 @@ def parse_args(args):
parents=[arv_cmd.retry_opt])
parser.add_argument('locator', type=str,
- help="""Collection UUID or locator""")
+ help="""Collection UUID or locator, optionally with a subdir path.""")
parser.add_argument('-s', action='store_true',
help="""List file sizes, in KiB.""")
parser.add_argument('--version', action='version',
@@ -45,9 +46,26 @@ def main(args, stdout, stderr, api_client=None, logger=None):
logger = logging.getLogger('arvados.arv-ls')
try:
- cr = arvados.CollectionReader(args.locator, api_client=api_client,
+ r = re.search(r'^(.*?)(/.*)?$', args.locator)
+ collection = r.group(1)
+ get_prefix = r.group(2)
+
+ cr = arvados.CollectionReader(collection, api_client=api_client,
num_retries=args.retries)
- except (arvados.errors.ArgumentError,
+ if get_prefix:
+ if get_prefix[-1] == '/':
+ get_prefix = get_prefix[:-1]
+ stream_name = '.' + get_prefix
+ reader = cr.find(stream_name)
+ if not (isinstance(reader, arvados.CollectionReader) or
+ isinstance(reader, arvados.collection.Subcollection)):
+ logger.error("'{}' is not a subdirectory".format(get_prefix))
+ return 1
+ else:
+ stream_name = '.'
+ reader = cr
+ except (arvados.errors.ApiError,
+ arvados.errors.ArgumentError,
arvados.errors.NotFoundError) as error:
logger.error("error fetching collection: {}".format(error))
return 1
@@ -57,7 +75,7 @@ def main(args, stdout, stderr, api_client=None, logger=None):
formatters.append(size_formatter)
formatters.append(name_formatter)
- for f in files_in_collection(cr):
+ for f in files_in_collection(reader, stream_name):
print(*(info_func(f) for info_func in formatters), file=stdout)
return 0
commit 2c0c6e6ba25ec56bab8c224865a7979af577adaa
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Mon Apr 3 17:45:34 2017 -0300
7824: Several improvements:
* Added error reporting on cases where the user ask for downloading a nonexisting file/subdir inside a collection.
* Enhanced error message when the user ask to download a non existing collection.
* Improved the collection traversal when downloading a subcollection.
* Exit with an error message when asking for a subcollection name without the trailing '/'.
* Error out if asking to get a subcollection with '-' as destination.
* Test updated.
diff --git a/sdk/cli/test/test_arv-keep-get.rb b/sdk/cli/test/test_arv-keep-get.rb
index 04f4543..b1f6bdf 100644
--- a/sdk/cli/test/test_arv-keep-get.rb
+++ b/sdk/cli/test/test_arv-keep-get.rb
@@ -140,7 +140,7 @@ class TestArvKeepGet < Minitest::Test
assert_arv_get false, 'e796ab2294f3e48ec709ffa8d6daf58c'
end
assert_equal '', out
- assert_match /Error:/, err
+ assert_match /ERROR:/, err
end
def test_nonexistent_manifest
@@ -148,7 +148,7 @@ class TestArvKeepGet < Minitest::Test
assert_arv_get false, 'acbd18db4cc2f85cedef654fccc4a4d8/', 'tmp/'
end
assert_equal '', out
- assert_match /Error:/, err
+ assert_match /ERROR:/, err
end
def test_manifest_root_to_dir
diff --git a/sdk/python/arvados/commands/get.py b/sdk/python/arvados/commands/get.py
index bf08441..c7254da 100755
--- a/sdk/python/arvados/commands/get.py
+++ b/sdk/python/arvados/commands/get.py
@@ -141,6 +141,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
logger.error("failed to read collection: {}".format(error))
return 1
+ # User asked to download the collection's manifest
if not get_prefix:
if not args.n:
open_flags = os.O_CREAT | os.O_WRONLY
@@ -166,28 +167,39 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
todo = []
todo_bytes = 0
try:
- for s, f in files_in_collection(reader):
- if get_prefix and get_prefix[-1] == os.sep:
- if not os.path.join(s.stream_name(),
- f.name).startswith('.' + get_prefix):
- continue
- if args.destination == "-":
- dest_path = "-"
- else:
- dest_path = os.path.join(
- args.destination,
- os.path.join(s.stream_name(), f.name)[len(get_prefix)+1:])
- if (not (args.n or args.f or args.skip_existing) and
- os.path.exists(dest_path)):
- logger.error('Local file %s already exists.' % (dest_path,))
- return 1
- else:
- if os.path.join(s.stream_name(), f.name) != '.' + get_prefix:
- continue
- dest_path = args.destination
- todo += [(s, f, dest_path)]
- todo_bytes += f.size()
- except arvados.errors.NotFoundError as e:
+ if get_prefix == os.sep:
+ item = reader
+ else:
+ item = reader.find('.' + get_prefix)
+
+ if isinstance(item, arvados.collection.Subcollection) or isinstance(item, arvados.collection.CollectionReader):
+ # If the user asked for a file and we got a subcollection, error out.
+ if get_prefix[-1] != os.sep:
+ logger.error("requested file '{}' is in fact a subcollection. Append a trailing '/' to download it.".format('.' + get_prefix))
+ return 1
+ # If the user asked stdout as a destination, error out.
+ elif args.destination == '-':
+ logger.error("cannot use 'stdout' as destination when downloading multiple files.")
+ return 1
+ # User asked for a subcollection, and that's what was found. Add up total size
+ # to download.
+ for s, f in files_in_collection(item):
+ dest_path = os.path.join(
+ args.destination,
+ os.path.join(s.stream_name(), f.name)[len(get_prefix)+1:])
+ if (not (args.n or args.f or args.skip_existing) and
+ os.path.exists(dest_path)):
+ logger.error('Local file %s already exists.' % (dest_path,))
+ return 1
+ todo += [(s, f, dest_path)]
+ todo_bytes += f.size()
+ elif isinstance(item, arvados.arvfile.ArvadosFile):
+ todo += [(item.parent, item, args.destination)]
+ todo_bytes += item.size()
+ else:
+ logger.error("'{}' not found.".format('.' + get_prefix))
+ return 1
+ except (IOError, arvados.errors.NotFoundError) as e:
logger.error(e)
return 1
@@ -244,7 +256,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
os.unlink(outfile.name)
break
finally:
- if outfile is not stdout:
+ if outfile != None and outfile != stdout:
outfile.close()
if args.progress:
commit 988e9052575bc93db2909f3b9ef576f1043eddcb
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Mon Apr 3 15:07:48 2017 -0300
7824: Log errors when trying to read the collection specified.
diff --git a/sdk/python/arvados/commands/get.py b/sdk/python/arvados/commands/get.py
index 2d3af97..bf08441 100755
--- a/sdk/python/arvados/commands/get.py
+++ b/sdk/python/arvados/commands/get.py
@@ -135,7 +135,11 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
get_prefix = r.group(2)
if args.r and not get_prefix:
get_prefix = os.sep
- reader = arvados.CollectionReader(collection, num_retries=args.retries)
+ try:
+ reader = arvados.CollectionReader(collection, num_retries=args.retries)
+ except Exception as error:
+ logger.error("failed to read collection: {}".format(error))
+ return 1
if not get_prefix:
if not args.n:
commit 3cc77494ac02e0f64a814cd4a898662fb7b329af
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Mon Apr 3 14:52:36 2017 -0300
7824: Closing file when finished writing
diff --git a/sdk/python/arvados/commands/get.py b/sdk/python/arvados/commands/get.py
index 67f38c4..2d3af97 100755
--- a/sdk/python/arvados/commands/get.py
+++ b/sdk/python/arvados/commands/get.py
@@ -164,8 +164,8 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
try:
for s, f in files_in_collection(reader):
if get_prefix and get_prefix[-1] == os.sep:
- if 0 != string.find(os.path.join(s.stream_name(), f.name),
- '.' + get_prefix):
+ if not os.path.join(s.stream_name(),
+ f.name).startswith('.' + get_prefix):
continue
if args.destination == "-":
dest_path = "-"
@@ -239,6 +239,9 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
if outfile and (outfile.fileno() > 2) and not outfile.closed:
os.unlink(outfile.name)
break
+ finally:
+ if outfile is not stdout:
+ outfile.close()
if args.progress:
stderr.write('\n')
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list