[ARVADOS] created: 2.1.0-1978-g062d4e06b
Git user
git at public.arvados.org
Mon Feb 28 21:35:08 UTC 2022
at 062d4e06b5003f348e055314fffdef525b5403cb (commit)
commit 062d4e06b5003f348e055314fffdef525b5403cb
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Mon Feb 28 18:03:02 2022 -0300
18574: Adds some more test cases on convert_to_* methods.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/python/tests/test_vocabulary.py b/sdk/python/tests/test_vocabulary.py
index a3b6391b0..003bd3209 100644
--- a/sdk/python/tests/test_vocabulary.py
+++ b/sdk/python/tests/test_vocabulary.py
@@ -59,6 +59,13 @@ class VocabularyTest(unittest.TestCase):
},
},
},
+ 'IDTAGCOMMENTS': {
+ 'strict': False,
+ 'labels': [
+ {'label': 'Comment'},
+ {'label': 'Notes'},
+ ],
+ },
},
}
@@ -72,7 +79,8 @@ class VocabularyTest(unittest.TestCase):
self.assertEqual(
self.voc.key_aliases.keys(),
set(['idtaganimals', 'creature', 'animal',
- 'idtagimportances', 'importance', 'priority'])
+ 'idtagimportances', 'importance', 'priority',
+ 'idtagcomments', 'comment', 'notes'])
)
vk = self.voc.key_aliases['creature']
@@ -127,15 +135,15 @@ class VocabularyTest(unittest.TestCase):
def test_convert_to_identifiers_multiple_pairs(self):
cases = [
- {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1'},
- {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1'},
- {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1'},
- {'priority': 'high priority', 'animal': 'IDVALANIMAL1'},
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+ {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1', 'comment': 'Very important person'},
+ {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1', 'notes': 'Very important person'},
+ {'priority': 'high priority', 'animal': 'IDVALANIMAL1', 'NOTES': 'Very important person'},
]
for case in cases:
self.assertEqual(
self.voc.convert_to_identifiers(case),
- {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1'},
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
"failing test case: {}".format(case)
)
@@ -174,18 +182,24 @@ class VocabularyTest(unittest.TestCase):
def test_convert_to_labels_multiple_pairs(self):
cases = [
- {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1'},
- {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1'},
- {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1'},
- {'priority': 'high priority', 'animal': 'IDVALANIMAL1'},
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+ {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1', 'comment': 'Very important person'},
+ {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1', 'notes': 'Very important person'},
+ {'priority': 'high priority', 'animal': 'IDVALANIMAL1', 'NOTES': 'Very important person'},
]
for case in cases:
self.assertEqual(
self.voc.convert_to_labels(case),
- {'Importance': 'High', 'Animal': 'Human'},
+ {'Importance': 'High', 'Animal': 'Human', 'Comment': 'Very important person'},
"failing test case: {}".format(case)
)
+ def test_convert_roundtrip(self):
+ initial = {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'}
+ converted = self.voc.convert_to_labels(initial)
+ self.assertNotEqual(converted, initial)
+ self.assertEqual(self.voc.convert_to_identifiers(converted), initial)
+
def test_convert_to_labels_unknown_key(self):
# Non-strict vocabulary
self.assertEqual(self.voc.strict_keys, False)
commit c355095d6c76cac068913e895914d444a980419f
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Mon Feb 28 17:51:29 2022 -0300
18574: Adds strict checking on conversion methods. Adds/updates tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/python/arvados/vocabulary.py b/sdk/python/arvados/vocabulary.py
index b4148890a..d51435f03 100644
--- a/sdk/python/arvados/vocabulary.py
+++ b/sdk/python/arvados/vocabulary.py
@@ -46,9 +46,11 @@ class Vocabulary(object):
try:
v_id = self[k][v].identifier
except KeyError:
- pass
+ if self[k].strict:
+ raise ValueError("value '%s' not found for key '%s'" % (v, k))
except KeyError:
- pass
+ if self.strict_keys:
+ raise KeyError("key '%s' not found" % k)
r[k_id] = v_id
return r
@@ -65,9 +67,11 @@ class Vocabulary(object):
try:
v_lbl = self[k][v].preferred_label
except KeyError:
- pass
+ if self[k].strict:
+ raise ValueError("value '%s' not found for key '%s'" % (v, k))
except KeyError:
- pass
+ if self.strict_keys:
+ raise KeyError("key '%s' not found" % k)
r[k_lbl] = v_lbl
return r
diff --git a/sdk/python/tests/test_vocabulary.py b/sdk/python/tests/test_vocabulary.py
index 7cca66a1c..a3b6391b0 100644
--- a/sdk/python/tests/test_vocabulary.py
+++ b/sdk/python/tests/test_vocabulary.py
@@ -32,7 +32,7 @@ class VocabularyTest(unittest.TestCase):
},
},
},
- 'IDTAGIMPORTANCE': {
+ 'IDTAGIMPORTANCES': {
'strict': True,
'labels': [
{'label': 'Importance'},
@@ -72,7 +72,7 @@ class VocabularyTest(unittest.TestCase):
self.assertEqual(
self.voc.key_aliases.keys(),
set(['idtaganimals', 'creature', 'animal',
- 'idtagimportance', 'importance', 'priority'])
+ 'idtagimportances', 'importance', 'priority'])
)
vk = self.voc.key_aliases['creature']
@@ -113,22 +113,55 @@ class VocabularyTest(unittest.TestCase):
def test_convert_to_identifiers(self):
cases = [
- {'IDTAGIMPORTANCE': 'IDVALIMPORTANCE1'},
- {'IDTAGIMPORTANCE': 'High'},
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+ {'IDTAGIMPORTANCES': 'High'},
{'importance': 'IDVALIMPORTANCE1'},
{'priority': 'high priority'},
]
for case in cases:
self.assertEqual(
self.voc.convert_to_identifiers(case),
- {'IDTAGIMPORTANCE': 'IDVALIMPORTANCE1'},
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
"failing test case: {}".format(case)
)
+ def test_convert_to_identifiers_multiple_pairs(self):
+ cases = [
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1'},
+ {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1'},
+ {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1'},
+ {'priority': 'high priority', 'animal': 'IDVALANIMAL1'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_identifiers(case),
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1'},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_identifiers_unknown_key(self):
+ # Non-strict vocabulary
+ self.assertEqual(self.voc.strict_keys, False)
+ self.assertEqual(self.voc.convert_to_identifiers({'foo': 'bar'}), {'foo': 'bar'})
+ # Strict vocabulary
+ strict_voc = arvados.vocabulary.Vocabulary(self.EXAMPLE_VOC)
+ strict_voc.strict_keys = True
+ with self.assertRaises(KeyError):
+ strict_voc.convert_to_identifiers({'foo': 'bar'})
+
+ def test_convert_to_identifiers_unknown_value(self):
+ # Non-strict key
+ self.assertEqual(self.voc['animal'].strict, False)
+ self.assertEqual(self.voc.convert_to_identifiers({'Animal': 'foo'}), {'IDTAGANIMALS': 'foo'})
+ # Strict key
+ self.assertEqual(self.voc['priority'].strict, True)
+ with self.assertRaises(ValueError):
+ self.voc.convert_to_identifiers({'Priority': 'foo'})
+
def test_convert_to_labels(self):
cases = [
- {'IDTAGIMPORTANCE': 'IDVALIMPORTANCE1'},
- {'IDTAGIMPORTANCE': 'High'},
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+ {'IDTAGIMPORTANCES': 'High'},
{'importance': 'IDVALIMPORTANCE1'},
{'priority': 'high priority'},
]
@@ -137,4 +170,37 @@ class VocabularyTest(unittest.TestCase):
self.voc.convert_to_labels(case),
{'Importance': 'High'},
"failing test case: {}".format(case)
- )
\ No newline at end of file
+ )
+
+ def test_convert_to_labels_multiple_pairs(self):
+ cases = [
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1'},
+ {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1'},
+ {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1'},
+ {'priority': 'high priority', 'animal': 'IDVALANIMAL1'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_labels(case),
+ {'Importance': 'High', 'Animal': 'Human'},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_labels_unknown_key(self):
+ # Non-strict vocabulary
+ self.assertEqual(self.voc.strict_keys, False)
+ self.assertEqual(self.voc.convert_to_labels({'foo': 'bar'}), {'foo': 'bar'})
+ # Strict vocabulary
+ strict_voc = arvados.vocabulary.Vocabulary(self.EXAMPLE_VOC)
+ strict_voc.strict_keys = True
+ with self.assertRaises(KeyError):
+ strict_voc.convert_to_labels({'foo': 'bar'})
+
+ def test_convert_to_labels_unknown_value(self):
+ # Non-strict key
+ self.assertEqual(self.voc['animal'].strict, False)
+ self.assertEqual(self.voc.convert_to_labels({'IDTAGANIMALS': 'foo'}), {'Animal': 'foo'})
+ # Strict key
+ self.assertEqual(self.voc['priority'].strict, True)
+ with self.assertRaises(ValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': 'foo'})
\ No newline at end of file
commit c68a6e131b0e2938d181467febedf2a50a1aa816
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Mon Feb 28 17:20:49 2022 -0300
18574: Adds conversion methods. Improves & adds tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/python/arvados/vocabulary.py b/sdk/python/arvados/vocabulary.py
index 1791566b5..b4148890a 100644
--- a/sdk/python/arvados/vocabulary.py
+++ b/sdk/python/arvados/vocabulary.py
@@ -26,13 +26,51 @@ class Vocabulary(object):
labels = [l['label'] for l in v_val.get('labels', [])]
values[v_id] = VocabularyValue(v_id, labels)
vk = VocabularyKey(key_id, key_labels, values, strict)
- self.key_aliases[key_id] = vk
+ self.key_aliases[key_id.lower()] = vk
for alias in vk.aliases:
self.key_aliases[alias.lower()] = vk
def __getitem__(self, key):
return self.key_aliases[key.lower()]
+ def convert_to_identifiers(self, obj={}):
+ """Translate key/value pairs to machine readable identifiers.
+ """
+ if not isinstance(obj, dict):
+ raise ValueError("obj must be a dict")
+ r = {}
+ for k, v in obj.items():
+ k_id, v_id = k, v
+ try:
+ k_id = self[k].identifier
+ try:
+ v_id = self[k][v].identifier
+ except KeyError:
+ pass
+ except KeyError:
+ pass
+ r[k_id] = v_id
+ return r
+
+ def convert_to_labels(self, obj={}):
+ """Translate key/value pairs to human readable labels.
+ """
+ if not isinstance(obj, dict):
+ raise ValueError("obj must be a dict")
+ r = {}
+ for k, v in obj.items():
+ k_lbl, v_lbl = k, v
+ try:
+ k_lbl = self[k].preferred_label
+ try:
+ v_lbl = self[k][v].preferred_label
+ except KeyError:
+ pass
+ except KeyError:
+ pass
+ r[k_lbl] = v_lbl
+ return r
+
class VocabularyData(object):
def __init__(self, identifier, aliases=[]):
self.identifier = identifier
@@ -53,7 +91,7 @@ class VocabularyKey(VocabularyData):
self.strict = strict
self.value_aliases = {}
for v_id, v_val in values.items():
- self.value_aliases[v_id] = v_val
+ self.value_aliases[v_id.lower()] = v_val
for v_alias in v_val.aliases:
self.value_aliases[v_alias.lower()] = v_val
diff --git a/sdk/python/tests/test_vocabulary.py b/sdk/python/tests/test_vocabulary.py
index ccaa7fe88..7cca66a1c 100644
--- a/sdk/python/tests/test_vocabulary.py
+++ b/sdk/python/tests/test_vocabulary.py
@@ -62,44 +62,79 @@ class VocabularyTest(unittest.TestCase):
},
}
- def perform_vocabulary_tests(self, voc):
- self.assertEqual(voc.strict_keys, False)
+ def setUp(self):
+ self.api = arvados.api('v1')
+ self.voc = vocabulary.Vocabulary(self.EXAMPLE_VOC)
+ self.api.vocabulary = mock.MagicMock(return_value=self.EXAMPLE_VOC)
+
+ def test_vocabulary_keys(self):
+ self.assertEqual(self.voc.strict_keys, False)
self.assertEqual(
- voc.key_aliases.keys(),
- set(['IDTAGANIMALS', 'creature', 'animal',
- 'IDTAGIMPORTANCE', 'importance', 'priority'])
+ self.voc.key_aliases.keys(),
+ set(['idtaganimals', 'creature', 'animal',
+ 'idtagimportance', 'importance', 'priority'])
)
- vk = voc.key_aliases['creature']
+ vk = self.voc.key_aliases['creature']
self.assertEqual(vk.strict, False)
self.assertEqual(vk.identifier, 'IDTAGANIMALS')
self.assertEqual(vk.aliases, ['Animal', 'Creature'])
self.assertEqual(vk.preferred_label, 'Animal')
+ self.assertEqual(
+ vk.value_aliases.keys(),
+ set(['idvalanimal1', 'human', 'homo sapiens',
+ 'idvalanimal2', 'elephant', 'loxodonta'])
+ )
+ def test_vocabulary_values(self):
+ vk = self.voc.key_aliases['creature']
vv = vk.value_aliases['human']
self.assertEqual(vv.identifier, 'IDVALANIMAL1')
self.assertEqual(vv.aliases, ['Human', 'Homo sapiens'])
self.assertEqual(vv.preferred_label, 'Human')
- self.assertEqual(voc['creature']['human'].identifier, vv.identifier)
- self.assertEqual(voc['Creature']['Human'].identifier, vv.identifier)
- self.assertEqual(voc['CREATURE']['HUMAN'].identifier, vv.identifier)
+ def test_vocabulary_indexing(self):
+ self.assertEqual(self.voc['creature']['human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(self.voc['Creature']['Human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(self.voc['CREATURE']['HUMAN'].identifier, 'IDVALANIMAL1')
with self.assertRaises(KeyError):
- inexistant = voc['foo']
+ inexistant = self.voc['foo']
def test_empty_vocabulary(self):
- voc = vocabulary.Vocabulary()
+ voc = vocabulary.Vocabulary({})
self.assertEqual(voc.strict_keys, False)
self.assertEqual(voc.key_aliases, {})
- def test_vocabulary_explicit_instantiation(self):
- voc = vocabulary.Vocabulary(self.EXAMPLE_VOC)
- self.perform_vocabulary_tests(voc)
+ def test_load_vocabulary_with_api(self):
+ voc = vocabulary.load_vocabulary(self.api)
+ self.assertEqual(voc['creature']['human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(voc['Creature']['Human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(voc['CREATURE']['HUMAN'].identifier, 'IDVALANIMAL1')
- @mock.patch('arvados.api')
- def test_load_vocabulary_with_api(self, api_mock):
- api_mock.return_value = mock.MagicMock()
- api_mock.return_value.vocabulary.return_value = self.EXAMPLE_VOC
+ def test_convert_to_identifiers(self):
+ cases = [
+ {'IDTAGIMPORTANCE': 'IDVALIMPORTANCE1'},
+ {'IDTAGIMPORTANCE': 'High'},
+ {'importance': 'IDVALIMPORTANCE1'},
+ {'priority': 'high priority'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_identifiers(case),
+ {'IDTAGIMPORTANCE': 'IDVALIMPORTANCE1'},
+ "failing test case: {}".format(case)
+ )
- voc = vocabulary.load_vocabulary(arvados.api('v1'))
- self.perform_vocabulary_tests(voc)
+ def test_convert_to_labels(self):
+ cases = [
+ {'IDTAGIMPORTANCE': 'IDVALIMPORTANCE1'},
+ {'IDTAGIMPORTANCE': 'High'},
+ {'importance': 'IDVALIMPORTANCE1'},
+ {'priority': 'high priority'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_labels(case),
+ {'Importance': 'High'},
+ "failing test case: {}".format(case)
+ )
\ No newline at end of file
commit 0c13e1261dffe71588a80f0af3f702cea194efc1
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Mon Feb 28 15:50:37 2022 -0300
18574: Adds 'preferred_label' virtual attribute. Avoids storing the voc twice.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/python/arvados/vocabulary.py b/sdk/python/arvados/vocabulary.py
index 643e167e6..1791566b5 100644
--- a/sdk/python/arvados/vocabulary.py
+++ b/sdk/python/arvados/vocabulary.py
@@ -15,8 +15,7 @@ def load_vocabulary(api_client=api('v1')):
class Vocabulary(object):
def __init__(self, voc_definition={}):
- self._definition = voc_definition
- self.strict_keys = self._definition.get('strict_tags', False)
+ self.strict_keys = voc_definition.get('strict_tags', False)
self.key_aliases = {}
for key_id, val in voc_definition.get('tags', {}).items():
@@ -39,6 +38,11 @@ class VocabularyData(object):
self.identifier = identifier
self.aliases = aliases
+ def __getattribute__(self, name):
+ if name == 'preferred_label':
+ return self.aliases[0]
+ return super(VocabularyData, self).__getattribute__(name)
+
class VocabularyValue(VocabularyData):
def __init__(self, identifier, aliases=[]):
super(VocabularyValue, self).__init__(identifier, aliases)
diff --git a/sdk/python/tests/test_vocabulary.py b/sdk/python/tests/test_vocabulary.py
index 49fd60126..ccaa7fe88 100644
--- a/sdk/python/tests/test_vocabulary.py
+++ b/sdk/python/tests/test_vocabulary.py
@@ -74,14 +74,18 @@ class VocabularyTest(unittest.TestCase):
self.assertEqual(vk.strict, False)
self.assertEqual(vk.identifier, 'IDTAGANIMALS')
self.assertEqual(vk.aliases, ['Animal', 'Creature'])
+ self.assertEqual(vk.preferred_label, 'Animal')
vv = vk.value_aliases['human']
self.assertEqual(vv.identifier, 'IDVALANIMAL1')
self.assertEqual(vv.aliases, ['Human', 'Homo sapiens'])
+ self.assertEqual(vv.preferred_label, 'Human')
self.assertEqual(voc['creature']['human'].identifier, vv.identifier)
self.assertEqual(voc['Creature']['Human'].identifier, vv.identifier)
self.assertEqual(voc['CREATURE']['HUMAN'].identifier, vv.identifier)
+ with self.assertRaises(KeyError):
+ inexistant = voc['foo']
def test_empty_vocabulary(self):
voc = vocabulary.Vocabulary()
commit 8dd0176878ad1ae38e14e09149bf9dcd812c5ce7
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Mon Feb 28 15:39:49 2022 -0300
18574: Fixes key_aliases/value_aliases indexing. Adds dict-style indexing.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/python/arvados/vocabulary.py b/sdk/python/arvados/vocabulary.py
index 8d89746ca..643e167e6 100644
--- a/sdk/python/arvados/vocabulary.py
+++ b/sdk/python/arvados/vocabulary.py
@@ -26,12 +26,18 @@ class Vocabulary(object):
for v_id, v_val in val.get('values', {}).items():
labels = [l['label'] for l in v_val.get('labels', [])]
values[v_id] = VocabularyValue(v_id, labels)
- self.key_aliases[key_id] = VocabularyKey(key_id, key_labels, values, strict)
+ vk = VocabularyKey(key_id, key_labels, values, strict)
+ self.key_aliases[key_id] = vk
+ for alias in vk.aliases:
+ self.key_aliases[alias.lower()] = vk
+
+ def __getitem__(self, key):
+ return self.key_aliases[key.lower()]
class VocabularyData(object):
def __init__(self, identifier, aliases=[]):
self.identifier = identifier
- self.aliases = set([x.lower() for x in aliases])
+ self.aliases = aliases
class VocabularyValue(VocabularyData):
def __init__(self, identifier, aliases=[]):
@@ -40,5 +46,12 @@ class VocabularyValue(VocabularyData):
class VocabularyKey(VocabularyData):
def __init__(self, identifier, aliases=[], values={}, strict=False):
super(VocabularyKey, self).__init__(identifier, aliases)
- self.values = values
- self.strict = strict
\ No newline at end of file
+ self.strict = strict
+ self.value_aliases = {}
+ for v_id, v_val in values.items():
+ self.value_aliases[v_id] = v_val
+ for v_alias in v_val.aliases:
+ self.value_aliases[v_alias.lower()] = v_val
+
+ def __getitem__(self, key):
+ return self.value_aliases[key.lower()]
\ No newline at end of file
diff --git a/sdk/python/tests/test_vocabulary.py b/sdk/python/tests/test_vocabulary.py
index 7aea129af..49fd60126 100644
--- a/sdk/python/tests/test_vocabulary.py
+++ b/sdk/python/tests/test_vocabulary.py
@@ -64,19 +64,31 @@ class VocabularyTest(unittest.TestCase):
def perform_vocabulary_tests(self, voc):
self.assertEqual(voc.strict_keys, False)
- self.assertEqual(voc.key_aliases.keys(), set(['IDTAGANIMALS', 'IDTAGIMPORTANCE']))
+ self.assertEqual(
+ voc.key_aliases.keys(),
+ set(['IDTAGANIMALS', 'creature', 'animal',
+ 'IDTAGIMPORTANCE', 'importance', 'priority'])
+ )
- self.assertEqual(voc.key_aliases['IDTAGANIMALS'].strict, False)
- self.assertEqual(set(voc.key_aliases['IDTAGANIMALS'].aliases), set(['animal', 'creature']))
- self.assertEqual(voc.key_aliases['IDTAGANIMALS'].values.keys(), set(['IDVALANIMAL1', 'IDVALANIMAL2']))
- self.assertEqual(voc.key_aliases['IDTAGANIMALS'].values['IDVALANIMAL1'].aliases, set(['human', 'homo sapiens']))
+ vk = voc.key_aliases['creature']
+ self.assertEqual(vk.strict, False)
+ self.assertEqual(vk.identifier, 'IDTAGANIMALS')
+ self.assertEqual(vk.aliases, ['Animal', 'Creature'])
+
+ vv = vk.value_aliases['human']
+ self.assertEqual(vv.identifier, 'IDVALANIMAL1')
+ self.assertEqual(vv.aliases, ['Human', 'Homo sapiens'])
+
+ self.assertEqual(voc['creature']['human'].identifier, vv.identifier)
+ self.assertEqual(voc['Creature']['Human'].identifier, vv.identifier)
+ self.assertEqual(voc['CREATURE']['HUMAN'].identifier, vv.identifier)
def test_empty_vocabulary(self):
voc = vocabulary.Vocabulary()
self.assertEqual(voc.strict_keys, False)
self.assertEqual(voc.key_aliases, {})
- def test_load_vocabulary(self):
+ def test_vocabulary_explicit_instantiation(self):
voc = vocabulary.Vocabulary(self.EXAMPLE_VOC)
self.perform_vocabulary_tests(voc)
commit 1b152f3f74c439938b5ed443e58e3a7b67266eec
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Mon Feb 28 13:14:32 2022 -0300
18574: Initial vocabulary loading support & tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/python/arvados/vocabulary.py b/sdk/python/arvados/vocabulary.py
new file mode 100644
index 000000000..8d89746ca
--- /dev/null
+++ b/sdk/python/arvados/vocabulary.py
@@ -0,0 +1,44 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+
+from . import api
+
+_logger = logging.getLogger('arvados.vocabulary')
+
+def load_vocabulary(api_client=api('v1')):
+ """Load the Arvados vocabulary from the API.
+ """
+ return Vocabulary(api_client.vocabulary())
+
+class Vocabulary(object):
+ def __init__(self, voc_definition={}):
+ self._definition = voc_definition
+ self.strict_keys = self._definition.get('strict_tags', False)
+ self.key_aliases = {}
+
+ for key_id, val in voc_definition.get('tags', {}).items():
+ strict = val.get('strict', False)
+ key_labels = [l['label'] for l in val.get('labels', [])]
+ values = {}
+ for v_id, v_val in val.get('values', {}).items():
+ labels = [l['label'] for l in v_val.get('labels', [])]
+ values[v_id] = VocabularyValue(v_id, labels)
+ self.key_aliases[key_id] = VocabularyKey(key_id, key_labels, values, strict)
+
+class VocabularyData(object):
+ def __init__(self, identifier, aliases=[]):
+ self.identifier = identifier
+ self.aliases = set([x.lower() for x in aliases])
+
+class VocabularyValue(VocabularyData):
+ def __init__(self, identifier, aliases=[]):
+ super(VocabularyValue, self).__init__(identifier, aliases)
+
+class VocabularyKey(VocabularyData):
+ def __init__(self, identifier, aliases=[], values={}, strict=False):
+ super(VocabularyKey, self).__init__(identifier, aliases)
+ self.values = values
+ self.strict = strict
\ No newline at end of file
diff --git a/sdk/python/tests/test_vocabulary.py b/sdk/python/tests/test_vocabulary.py
new file mode 100644
index 000000000..7aea129af
--- /dev/null
+++ b/sdk/python/tests/test_vocabulary.py
@@ -0,0 +1,89 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import unittest
+import mock
+
+from arvados import api, vocabulary
+
+class VocabularyTest(unittest.TestCase):
+ EXAMPLE_VOC = {
+ 'tags': {
+ 'IDTAGANIMALS': {
+ 'strict': False,
+ 'labels': [
+ {'label': 'Animal'},
+ {'label': 'Creature'},
+ ],
+ 'values': {
+ 'IDVALANIMAL1': {
+ 'labels': [
+ {'label': 'Human'},
+ {'label': 'Homo sapiens'},
+ ],
+ },
+ 'IDVALANIMAL2': {
+ 'labels': [
+ {'label': 'Elephant'},
+ {'label': 'Loxodonta'},
+ ],
+ },
+ },
+ },
+ 'IDTAGIMPORTANCE': {
+ 'strict': True,
+ 'labels': [
+ {'label': 'Importance'},
+ {'label': 'Priority'},
+ ],
+ 'values': {
+ 'IDVALIMPORTANCE1': {
+ 'labels': [
+ {'label': 'High'},
+ {'label': 'High priority'},
+ ],
+ },
+ 'IDVALIMPORTANCE2': {
+ 'labels': [
+ {'label': 'Medium'},
+ {'label': 'Medium priority'},
+ ],
+ },
+ 'IDVALIMPORTANCE3': {
+ 'labels': [
+ {'label': 'Low'},
+ {'label': 'Low priority'},
+ ],
+ },
+ },
+ },
+ },
+ }
+
+ def perform_vocabulary_tests(self, voc):
+ self.assertEqual(voc.strict_keys, False)
+ self.assertEqual(voc.key_aliases.keys(), set(['IDTAGANIMALS', 'IDTAGIMPORTANCE']))
+
+ self.assertEqual(voc.key_aliases['IDTAGANIMALS'].strict, False)
+ self.assertEqual(set(voc.key_aliases['IDTAGANIMALS'].aliases), set(['animal', 'creature']))
+ self.assertEqual(voc.key_aliases['IDTAGANIMALS'].values.keys(), set(['IDVALANIMAL1', 'IDVALANIMAL2']))
+ self.assertEqual(voc.key_aliases['IDTAGANIMALS'].values['IDVALANIMAL1'].aliases, set(['human', 'homo sapiens']))
+
+ def test_empty_vocabulary(self):
+ voc = vocabulary.Vocabulary()
+ self.assertEqual(voc.strict_keys, False)
+ self.assertEqual(voc.key_aliases, {})
+
+ def test_load_vocabulary(self):
+ voc = vocabulary.Vocabulary(self.EXAMPLE_VOC)
+ self.perform_vocabulary_tests(voc)
+
+ @mock.patch('arvados.api')
+ def test_load_vocabulary_with_api(self, api_mock):
+ api_mock.return_value = mock.MagicMock()
+ api_mock.return_value.vocabulary.return_value = self.EXAMPLE_VOC
+
+ voc = vocabulary.load_vocabulary(arvados.api('v1'))
+ self.perform_vocabulary_tests(voc)
commit c2158b90ffff32653196539d694e4b5a48dd01ae
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Wed Feb 23 15:40:12 2022 -0300
18574: Adds vocabulary endpoint section to the API documentation.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/doc/api/index.html.textile.liquid b/doc/api/index.html.textile.liquid
index 8586a166d..3d69d02ea 100644
--- a/doc/api/index.html.textile.liquid
+++ b/doc/api/index.html.textile.liquid
@@ -20,6 +20,10 @@ h2. Exported configuration
The Controller exposes a subset of the cluster's configuration and makes it available to clients in JSON format. This public config includes valuable information like several service's URLs, timeout settings, etc. and it is available at @/arvados/v1/config@, for example @https://{{ site.arvados_api_host }}/arvados/v1/config at . The new Workbench is one example of a client using this information, as it's a client-side application and doesn't have access to the cluster's config file.
+h2. Exported vocabulary definition
+
+When configured, the Controller also exports the "metadata vocabulary definition":{{site.baseurl}}/admin/metadata-vocabulary.html in JSON format. This functionality is useful for clients like Workbench2 and the Python SDK to provide "identifier to human-readable labels" translations facilities for reading and writing objects on the system. This is available at @/arvados/v1/vocabulary@, for example @https://{{ site.arvados_api_host }}/arvados/v1/vocabulary at .
+
h2. Workbench examples
Many Arvados Workbench pages, under the *Advanced* tab, provide examples of API and SDK use for accessing the current resource .
commit 82c424076577660d96173213a0d2db5f7c1450d7
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Tue Feb 22 14:58:29 2022 -0300
18574: Adds /arvados/v1/vocabulary to the discovery document.
Also, adds vocabulary caching on PySDK's API client.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 88596211d..e0d1c50f0 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -253,6 +253,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False,
svc.insecure = insecure
svc.request_id = request_id
svc.config = lambda: util.get_config_once(svc)
+ svc.vocabulary = lambda: util.get_vocabulary_once(svc)
kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
kwargs['http'].cache = None
kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id()
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 2380e48b7..be8a03fc3 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -491,3 +491,11 @@ def get_config_once(svc):
if not hasattr(svc, '_cached_config'):
svc._cached_config = svc.configs().get().execute()
return svc._cached_config
+
+def get_vocabulary_once(svc):
+ if not svc._rootDesc.get('resources').get('vocabularies', False):
+ # Old API server version, no vocabulary export endpoint
+ return {}
+ if not hasattr(svc, '_cached_vocabulary'):
+ svc._cached_vocabulary = svc.vocabularies().get().execute()
+ return svc._cached_vocabulary
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 59ac639ba..100b91681 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -37,7 +37,7 @@ class Arvados::V1::SchemaController < ApplicationController
# format is YYYYMMDD, must be fixed width (needs to be lexically
# sortable), updated manually, may be used by clients to
# determine availability of API server features.
- revision: "20210628",
+ revision: "20220222",
source_version: AppVersion.hash,
sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
packageVersion: AppVersion.package_version,
@@ -427,6 +427,27 @@ class Arvados::V1::SchemaController < ApplicationController
}
}
+ discovery[:resources]['vocabularies'] = {
+ methods: {
+ get: {
+ id: "arvados.vocabularies.get",
+ path: "vocabulary",
+ httpMethod: "GET",
+ description: "Get vocabulary definition",
+ parameters: {
+ },
+ parameterOrder: [
+ ],
+ response: {
+ },
+ scopes: [
+ "https://api.arvados.org/auth/arvados",
+ "https://api.arvados.org/auth/arvados.readonly"
+ ]
+ },
+ }
+ }
+
discovery[:resources]['sys'] = {
methods: {
get: {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list