[ARVADOS] created: 2.1.0-1215-g5ac636b5a
Git user
git at public.arvados.org
Thu Aug 19 22:27:55 UTC 2021
at 5ac636b5a34e4b8bf6c32266a626575ff482878d (commit)
commit 5ac636b5a34e4b8bf6c32266a626575ff482878d
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Aug 19 19:27:12 2021 -0300
17696: Updates API documentation adding storage_classes_* fields.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/doc/api/methods/collections.html.textile.liquid b/doc/api/methods/collections.html.textile.liquid
index fd4a36f29..6c1cc691c 100644
--- a/doc/api/methods/collections.html.textile.liquid
+++ b/doc/api/methods/collections.html.textile.liquid
@@ -32,7 +32,10 @@ table(table table-bordered table-condensed).
|manifest_text|text|||
|replication_desired|number|Minimum storage replication level desired for each data block referenced by this collection. A value of @null@ signifies that the site default replication level (typically 2) is desired.|@2@|
|replication_confirmed|number|Replication level most recently confirmed by the storage system. This field is null when a collection is first created, and is reset to null when the manifest_text changes in a way that introduces a new data block. An integer value indicates the replication level of the _least replicated_ data block in the collection.|@2@, null|
-|replication_confirmed_at|datetime|When replication_confirmed was confirmed. If replication_confirmed is null, this field is also null.||
+|replication_confirmed_at|datetime|When @replication_confirmed@ was confirmed. If @replication_confirmed@ is null, this field is also null.||
+|storage_classes_desired|list|An optional list of storage class names where the blocks should be saved. If not provided, the cluster's default storage class(es) will be set.|@['archival']@|
+|storage_classes_confirmed|list|Storage classes most recently confirmed by the storage system. This field is an empty list when a collection is first created.|@'archival']@, @[]@|
+|storage_classes_confirmed_at|datetime|When @storage_classes_confirmed@ was confirmed. If @storage_classes_confirmed@ is @[]@, this field is null.||
|trash_at|datetime|If @trash_at@ is non-null and in the past, this collection will be hidden from API calls. May be untrashed.||
|delete_at|datetime|If @delete_at@ is non-null and in the past, the collection may be permanently deleted.||
|is_trashed|boolean|True if @trash_at@ is in the past, false if not.||
commit b99accd2cf9776d7025e27a12e8caff2af4df2ca
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Aug 19 18:29:19 2021 -0300
17696: Updates docstring on arvados.Collection class.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 44d032535..50cb703a5 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1296,8 +1296,8 @@ class Collection(RichCollectionBase):
:storage_classes_desired:
A list of storage class names where to upload the data. If None,
- the keepstores are expected to store the data into their default
- storage class.
+ the keep client is expected to store the data into the cluster's
+ default storage class(es).
"""
commit c091641eb5484b3cd61e34529529ced9f8ede148
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Aug 19 18:08:39 2021 -0300
17696: KeepClient uses default storage classes when none is required explicitly
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 86b1d91b8..9dfe0436d 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -842,6 +842,7 @@ class KeepClient(object):
self.hits_counter = Counter()
self.misses_counter = Counter()
self._storage_classes_unsupported_warning = False
+ self._default_classes = []
if local_store:
self.local_store = local_store
@@ -882,6 +883,12 @@ class KeepClient(object):
self._writable_services = None
self.using_proxy = None
self._static_services_list = False
+ try:
+ self._default_classes = [
+ k for k, v in self.api_client.config()['StorageClasses'].items() if v['Default']]
+ except KeyError:
+ # We're talking to an old cluster
+ pass
def current_timeout(self, attempt_number):
"""Return the appropriate timeout to use for this client.
@@ -1174,7 +1181,7 @@ class KeepClient(object):
"failed to read {} after {}".format(loc_s, loop.attempts_str()), service_errors, label="service")
@retry.retry_method
- def put(self, data, copies=2, num_retries=None, request_id=None, classes=[]):
+ def put(self, data, copies=2, num_retries=None, request_id=None, classes=None):
"""Save data in Keep.
This method will get a list of Keep services from the API server, and
@@ -1195,6 +1202,8 @@ class KeepClient(object):
be written.
"""
+ classes = classes or self._default_classes
+
if not isinstance(data, bytes):
data = data.encode()
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index f251ea654..d9b3ca86c 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -190,7 +190,13 @@ class MockStreamReader(object):
class ApiClientMock(object):
def api_client_mock(self):
- return mock.MagicMock(name='api_client_mock')
+ api_mock = mock.MagicMock(name='api_client_mock')
+ api_mock.config.return_value = {
+ 'StorageClasses': {
+ 'default': {'Default': True}
+ }
+ }
+ return api_mock
def mock_keep_services(self, api_mock=None, status=200, count=12,
service_type='disk',
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 0eefa586d..b1c42fd2b 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -540,26 +540,49 @@ class KeepStorageClassesTestCase(unittest.TestCase, tutil.ApiClientMock):
self.data = b'xyzzy'
self.locator = '1271ed5ef305aadabc605b1609e24c52'
+ def test_multiple_default_storage_classes_req_header(self):
+ api_mock = self.api_client_mock()
+ api_mock.config.return_value = {
+ 'StorageClasses': {
+ 'foo': { 'Default': True },
+ 'bar': { 'Default': True },
+ 'baz': { 'Default': False }
+ }
+ }
+ api_client = self.mock_keep_services(api_mock=api_mock, count=2)
+ keep_client = arvados.KeepClient(api_client=api_client)
+ resp_hdr = {
+ 'x-keep-storage-classes-confirmed': 'foo=1, bar=1',
+ 'x-keep-replicas-stored': 1
+ }
+ with tutil.mock_keep_responses(self.locator, 200, **resp_hdr) as mock:
+ keep_client.put(self.data, copies=1)
+ req_hdr = mock.responses[0]
+ self.assertIn(
+ 'X-Keep-Storage-Classes: bar, foo', req_hdr.getopt(pycurl.HTTPHEADER))
+
def test_storage_classes_req_header(self):
+ self.assertEqual(
+ self.api_client.config()['StorageClasses'],
+ {'default': {'Default': True}})
cases = [
# requested, expected
[['foo'], 'X-Keep-Storage-Classes: foo'],
[['bar', 'foo'], 'X-Keep-Storage-Classes: bar, foo'],
- [[], None],
+ [[], 'X-Keep-Storage-Classes: default'],
+ [None, 'X-Keep-Storage-Classes: default'],
]
for req_classes, expected_header in cases:
headers = {'x-keep-replicas-stored': 1}
- if len(req_classes) > 0:
+ if req_classes is None or len(req_classes) == 0:
+ confirmed_hdr = 'default=1'
+ elif len(req_classes) > 0:
confirmed_hdr = ', '.join(["{}=1".format(cls) for cls in req_classes])
- headers.update({'x-keep-storage-classes-confirmed': confirmed_hdr})
+ headers.update({'x-keep-storage-classes-confirmed': confirmed_hdr})
with tutil.mock_keep_responses(self.locator, 200, **headers) as mock:
self.keep_client.put(self.data, copies=1, classes=req_classes)
- resp = mock.responses[0]
- if expected_header is not None:
- self.assertIn(expected_header, resp.getopt(pycurl.HTTPHEADER))
- else:
- for hdr in resp.getopt(pycurl.HTTPHEADER):
- self.assertNotRegex(hdr, r'^X-Keep-Storage-Classes.*')
+ req_hdr = mock.responses[0]
+ self.assertIn(expected_header, req_hdr.getopt(pycurl.HTTPHEADER))
def test_partial_storage_classes_put(self):
headers = {
@@ -1368,6 +1391,8 @@ class KeepClientAPIErrorTest(unittest.TestCase):
return "abc"
elif r == "insecure":
return False
+ elif r == "config":
+ return lambda: {}
else:
raise arvados.errors.KeepReadError()
keep_client = arvados.KeepClient(api_client=ApiMock(),
commit 5963d701ce4bdc85847a46d22251493a737e9ff5
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Aug 19 14:33:20 2021 -0300
17696: Changes StorageClasses.SAMPLE.Default to false to avoid confusion.
Having it be 'true' can confuse readers to think that the 'Default' setting
is 'true' by default when the zero-value of boolean are 'false'.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index ec3261390..155c18b07 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -1250,7 +1250,7 @@ Clusters:
#
# If any StorageClasses are configured, at least one of them
# must have Default: true.
- Default: true
+ Default: false
Volumes:
SAMPLE:
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 36a21d4c3..31b571e15 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -1256,7 +1256,7 @@ Clusters:
#
# If any StorageClasses are configured, at least one of them
# must have Default: true.
- Default: true
+ Default: false
Volumes:
SAMPLE:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list