[ARVADOS] created: 1.1.4-284-gf0d40c1
Git user
git at public.curoverse.com
Tue May 22 06:48:48 EDT 2018
at f0d40c15b43521997703ba081f8ebded80a6e948 (commit)
commit f0d40c15b43521997703ba081f8ebded80a6e948
Author: Fuad Muhic <muhic.fuad at gmail.com>
Date: Tue May 22 12:25:28 2018 +0200
Added tests to arv-put to cover storage classes feature.
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 4b1f694..93cfdc2 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -730,6 +730,11 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers,
self.call_main_with_args,
['--project-uuid', self.Z_UUID, '--stream'])
+ def test_error_when_multiple_storage_classes_specified(self):
+ self.assertRaises(SystemExit,
+ self.call_main_with_args,
+ ['--storage-classes', 'hot,cold'])
+
def test_error_when_excluding_absolute_path(self):
tmpdir = self.make_tmpdir()
self.assertRaises(SystemExit,
@@ -1061,6 +1066,18 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
'--project-uuid', self.PROJECT_UUID])
self.assertEqual(link_name, collection['name'])
+ def test_put_collection_with_storage_classes_specified(self):
+ collection = self.run_and_find_collection("", ['--storage-classes', 'hot'])
+
+ self.assertEqual(len(collection['storage_classes_desired']), 1)
+ self.assertEqual(collection['storage_classes_desired'][0], 'hot')
+
+ def test_put_collection_without_storage_classes_specified(self):
+ collection = self.run_and_find_collection("")
+
+ self.assertEqual(len(collection['storage_classes_desired']), 1)
+ self.assertEqual(collection['storage_classes_desired'][0], 'default')
+
def test_exclude_filename_pattern(self):
tmpdir = self.make_tmpdir()
tmpsubdir = os.path.join(tmpdir, 'subdir')
commit 49bfee2a221bb050732935c240b981b7becd9aff
Author: Fuad Muhic <muhic.fuad at gmail.com>
Date: Mon May 21 17:18:37 2018 +0200
Added --storage-classes argument to arv-put.
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 8fb90c9..cce7d75 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1436,7 +1436,7 @@ class Collection(RichCollectionBase):
@must_be_writable
@synchronized
@retry_method
- def save(self, merge=True, num_retries=None):
+ def save(self, storage_classes=None, merge=True, num_retries=None):
"""Save collection to an existing collection record.
Commit pending buffer blocks to Keep, merge with remote record (if
@@ -1465,9 +1465,13 @@ class Collection(RichCollectionBase):
self.update()
text = self.manifest_text(strip=False)
+ body={'manifest_text': text}
+ if storage_classes:
+ body["storage_classes_desired"] = storage_classes
+
self._remember_api_response(self._my_api().collections().update(
uuid=self._manifest_locator,
- body={'manifest_text': text}
+ body=body
).execute(
num_retries=num_retries))
self._manifest_text = self._api_response["manifest_text"]
@@ -1483,6 +1487,7 @@ class Collection(RichCollectionBase):
def save_new(self, name=None,
create_collection_record=True,
owner_uuid=None,
+ storage_classes=None,
ensure_unique_name=False,
num_retries=None):
"""Save collection to a new collection record.
@@ -1525,6 +1530,8 @@ class Collection(RichCollectionBase):
"replication_desired": self.replication_desired}
if owner_uuid:
body["owner_uuid"] = owner_uuid
+ if storage_classes:
+ body["storage_classes_desired"] = storage_classes
self._remember_api_response(self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries))
text = self._api_response["manifest_text"]
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 388d87b..cba00c3 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -140,6 +140,10 @@ physical storage devices (e.g., disks) should have a copy of each data
block. Default is to use the server-provided default (if any) or 2.
""")
+upload_opts.add_argument('--storage-classes', help="""
+Specify comma separated list of storage classes to be used when saving data to Keep.
+""")
+
upload_opts.add_argument('--threads', type=int, metavar='N', default=None,
help="""
Set the number of upload threads to be used. Take into account that
@@ -418,8 +422,8 @@ class ArvPutUploadJob(object):
def __init__(self, paths, resume=True, use_cache=True, reporter=None,
name=None, owner_uuid=None, api_client=None,
ensure_unique_name=False, num_retries=None,
- put_threads=None, replication_desired=None,
- filename=None, update_time=60.0, update_collection=None,
+ put_threads=None, replication_desired=None, filename=None,
+ update_time=60.0, update_collection=None, storage_classes=None,
logger=logging.getLogger('arvados.arv_put'), dry_run=False,
follow_links=True, exclude_paths=[], exclude_names=None):
self.paths = paths
@@ -439,6 +443,7 @@ class ArvPutUploadJob(object):
self.replication_desired = replication_desired
self.put_threads = put_threads
self.filename = filename
+ self.storage_classes = storage_classes
self._api_client = api_client
self._state_lock = threading.Lock()
self._state = None # Previous run state (file list & manifest)
@@ -614,10 +619,14 @@ class ArvPutUploadJob(object):
else:
# The file already exist on remote collection, skip it.
pass
- self._remote_collection.save(num_retries=self.num_retries)
+ self._remote_collection.save(storage_classes=self.storage_classes,
+ num_retries=self.num_retries)
else:
+ if self.storage_classes is None:
+ self.storage_classes = ['default']
self._local_collection.save_new(
name=self.name, owner_uuid=self.owner_uuid,
+ storage_classes=self.storage_classes,
ensure_unique_name=self.ensure_unique_name,
num_retries=self.num_retries)
@@ -1045,6 +1054,15 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr,
else:
reporter = None
+ # Split storage-classes argument
+ storage_classes = None
+ if args.storage_classes:
+ storage_classes = args.storage_classes.strip().split(',')
+ if len(storage_classes) > 1:
+ logger.error("Multiple storage classes are not supported currently.")
+ sys.exit(1)
+
+
# Setup exclude regex from all the --exclude arguments provided
name_patterns = []
exclude_paths = []
@@ -1102,6 +1120,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr,
owner_uuid = project_uuid,
ensure_unique_name = True,
update_collection = args.update_collection,
+ storage_classes=storage_classes,
logger=logger,
dry_run=args.dry_run,
follow_links=args.follow_links,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list