[ARVADOS] created: b4a26f3454b98eb4cdc52ebbf8d4d00c67f5e49a
git at public.curoverse.com
git at public.curoverse.com
Fri Dec 12 18:02:09 EST 2014
at b4a26f3454b98eb4cdc52ebbf8d4d00c67f5e49a (commit)
commit b4a26f3454b98eb4cdc52ebbf8d4d00c67f5e49a
Author: Brett Smith <brett at curoverse.com>
Date: Fri Dec 12 18:01:39 2014 -0500
4792: arv-ls normalizes the collection before listing.
I sort of ended up rewriting arv-ls to make it testable, but hey,
that's part of the support task. Normalization is the only functional
change I made.
diff --git a/sdk/python/arvados/commands/ls.py b/sdk/python/arvados/commands/ls.py
new file mode 100755
index 0000000..e87244d
--- /dev/null
+++ b/sdk/python/arvados/commands/ls.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import argparse
+
+import arvados
+import arvados.commands._util as arv_cmd
+
+def parse_args(args):
+ parser = argparse.ArgumentParser(
+ description='List contents of a manifest',
+ parents=[arv_cmd.retry_opt])
+
+ parser.add_argument('locator', type=str,
+ help="""Collection UUID or locator""")
+ parser.add_argument('-s', action='store_true',
+ help="""List file sizes, in KiB.""")
+
+ return parser.parse_args(args)
+
+def size_formatter(coll_file):
+ return "{:>10}".format((coll_file.size() + 1023) / 1024)
+
+def name_formatter(coll_file):
+ return "{}/{}".format(coll_file.stream_name(), coll_file.name)
+
+def main(args, stdout, stderr, api_client=None):
+ args = parse_args(args)
+
+ if api_client is None:
+ api_client = arvados.api('v1')
+
+ try:
+ cr = arvados.CollectionReader(args.locator, api_client=api_client,
+ num_retries=args.retries)
+ cr.normalize()
+ except (arvados.errors.ArgumentError,
+ arvados.errors.NotFoundError) as error:
+ print("arv-ls: error fetching collection: {}".format(error),
+ file=stderr)
+ return 1
+
+ formatters = []
+ if args.s:
+ formatters.append(size_formatter)
+ formatters.append(name_formatter)
+
+ for f in cr.all_files():
+ print(*(info_func(f) for info_func in formatters), file=stdout)
+
+ return 0
diff --git a/sdk/python/bin/arv-ls b/sdk/python/bin/arv-ls
index 382bfe8..23b99f2 100755
--- a/sdk/python/bin/arv-ls
+++ b/sdk/python/bin/arv-ls
@@ -1,24 +1,7 @@
#!/usr/bin/env python
-import argparse
+import sys
-import arvados
-import arvados.commands._util as arv_cmd
+from arvados.commands.ls import main
-parser = argparse.ArgumentParser(
- description='List contents of a manifest',
- parents=[arv_cmd.retry_opt])
-
-parser.add_argument('locator', type=str,
- help="Collection UUID or locator")
-parser.add_argument('-s', action='store_true', help="""List file sizes, in KiB.""")
-
-args = parser.parse_args()
-cr = arvados.CollectionReader(args.locator, num_retries=args.retries)
-
-for f in cr.all_files():
- if args.s:
- print "{:>10} {}".format((f.size() + 1023) / 1024,
- f.stream_name() + "/" + f.name())
- else:
- print f.stream_name() + "/" + f.name()
+sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr))
diff --git a/sdk/python/tests/test_arv_ls.py b/sdk/python/tests/test_arv_ls.py
new file mode 100644
index 0000000..90bbacf
--- /dev/null
+++ b/sdk/python/tests/test_arv_ls.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import hashlib
+import io
+import random
+
+import mock
+
+import arvados.errors as arv_error
+import arvados.commands.ls as arv_ls
+import run_test_server
+
+class ArvLsTestCase(run_test_server.TestCaseWithServers):
+ FAKE_UUID = 'zzzzz-4zz18-12345abcde12345'
+
+ def newline_join(self, seq):
+ return '\n'.join(seq) + '\n'
+
+ def random_blocks(self, *sizes):
+ return ' '.join('{:032x}+{:d}'.format(
+ random.randint(0, (16 ** 32) - 1), size
+ ) for size in sizes)
+
+ def mock_api_for_manifest(self, manifest_lines, uuid=FAKE_UUID):
+ manifest_text = self.newline_join(manifest_lines)
+ pdh = '{}+{}'.format(hashlib.md5(manifest_text).hexdigest(),
+ len(manifest_text))
+ coll_info = {'uuid': uuid,
+ 'portable_data_hash': pdh,
+ 'manifest_text': manifest_text}
+ api_client = mock.MagicMock(name='mock_api_client')
+ api_client.collections().get().execute.return_value = coll_info
+ return coll_info, api_client
+
+ def run_ls(self, args, api_client):
+ self.stdout = io.BytesIO()
+ self.stderr = io.BytesIO()
+ return arv_ls.main(args, self.stdout, self.stderr, api_client)
+
+ def test_plain_listing(self):
+ collection, api_client = self.mock_api_for_manifest(
+ ['. {} 0:3:one.txt 3:4:two.txt'.format(self.random_blocks(5, 2)),
+ './dir {} 1:5:sub.txt'.format(self.random_blocks(8))])
+ self.assertEqual(0, self.run_ls([collection['uuid']], api_client))
+ self.assertEqual(
+ self.newline_join(['./one.txt', './two.txt', './dir/sub.txt']),
+ self.stdout.getvalue())
+ self.assertEqual('', self.stderr.getvalue())
+
+ def test_size_listing(self):
+ collection, api_client = self.mock_api_for_manifest(
+ ['. {} 0:0:0.txt 0:1000:1.txt 1000:2000:2.txt'.format(
+ self.random_blocks(3000))])
+ self.assertEqual(0, self.run_ls(['-s', collection['uuid']], api_client))
+ self.stdout.seek(0, 0)
+ for expected in range(3):
+ actual_size, actual_name = self.stdout.readline().split()
+ # But she seems much bigger to me...
+ self.assertEqual(str(expected), actual_size)
+ self.assertEqual('./{}.txt'.format(expected), actual_name)
+ self.assertEqual('', self.stdout.read(-1))
+ self.assertEqual('', self.stderr.getvalue())
+
+ def test_nonnormalized_manifest(self):
+ collection, api_client = self.mock_api_for_manifest(
+ ['. {} 0:1010:non.txt'.format(self.random_blocks(1010)),
+ '. {} 0:2020:non.txt'.format(self.random_blocks(2020))])
+ self.assertEqual(0, self.run_ls(['-s', collection['uuid']], api_client))
+ self.stdout.seek(0, 0)
+ self.assertEqual(['3', './non.txt'], self.stdout.readline().split())
+ self.assertEqual('', self.stdout.read(-1))
+ self.assertEqual('', self.stderr.getvalue())
+
+ def test_locator_failure(self):
+ api_client = mock.MagicMock(name='mock_api_client')
+ api_client.collections().get().execute.side_effect = (
+ arv_error.NotFoundError)
+ self.assertNotEqual(0, self.run_ls([self.FAKE_UUID], api_client))
+ self.assertNotEqual('', self.stderr.getvalue())
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list