[ARVADOS] updated: 2.1.0-112-g9432f0013
Git user
git at public.arvados.org
Tue Nov 24 23:00:37 UTC 2020
Summary of changes:
doc/sdk/python/cookbook.html.textile.liquid | 31 +++++++
sdk/python/tests/test_util.py | 137 ++++++++++++++++++++++++++++
2 files changed, 168 insertions(+)
via 9432f0013765366507329649a5b8e1ea6345aaa2 (commit)
via 872584d6c225c5ca1d20dcf6b0fc74c3df6a1257 (commit)
from 51d712fe2f3e46fe6b2358bb20196c6d4e4e26a3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9432f0013765366507329649a5b8e1ea6345aaa2
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 24 18:00:12 2020 -0500
17022: Add example usage of keyset_list_all to code cookbook.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/sdk/python/cookbook.html.textile.liquid b/doc/sdk/python/cookbook.html.textile.liquid
index 82741c3ea..3aa01bbb5 100644
--- a/doc/sdk/python/cookbook.html.textile.liquid
+++ b/doc/sdk/python/cookbook.html.textile.liquid
@@ -257,3 +257,34 @@ for f in files_to_copy:
target.save_new(name=target_name, owner_uuid=target_project)
print("Created collection %s" % target.manifest_locator())
{% endcodeblock %}
+
+h2. Copy files from a collection another collection
+
+{% codeblock as python %}
+import arvados.collection
+
+source_collection = "x1u39-4zz18-krzg64ufvehgitl"
+target_collection = "x1u39-4zz18-67q94einb8ptznm"
+files_to_copy = ["folder1/sample1/sample1_R1.fastq",
+ "folder1/sample2/sample2_R1.fastq"]
+
+source = arvados.collection.CollectionReader(source_collection)
+target = arvados.collection.Collection(target_collection)
+
+for f in files_to_copy:
+ target.copy(f, "", source_collection=source)
+
+target.save()
+{% endcodeblock %}
+
+h2. Listing records with paging
+
+Use the @arvados.util.keyset_list_all@ helper method to iterate over all the records matching an optional filter. This method handles paging internally and returns results incrementally using a Python iterator. The first parameter of the method takes a @list@ method of an Arvados resource (@collections@, @container_requests@, etc).
+
+{% codeblock as python %}
+import arvados.util
+
+api = arvados.api()
+for c in arvados.util.keyset_list_all(api.collections().list, filters=[["name", "like", "%sample123%"]]):
+ print("got collection " + c["uuid"])
+{% endcodeblock %}
commit 872584d6c225c5ca1d20dcf6b0fc74c3df6a1257
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 24 17:51:21 2020 -0500
17022: Add test cases for keyset_list_all
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/python/tests/test_util.py b/sdk/python/tests/test_util.py
index 87074dbdf..1c0e437b4 100644
--- a/sdk/python/tests/test_util.py
+++ b/sdk/python/tests/test_util.py
@@ -7,6 +7,7 @@ import subprocess
import unittest
import arvados
+import arvados.util
class MkdirDashPTest(unittest.TestCase):
def setUp(self):
@@ -38,3 +39,139 @@ class RunCommandTestCase(unittest.TestCase):
def test_failure(self):
with self.assertRaises(arvados.errors.CommandFailedError):
arvados.util.run_command(['false'])
+
+class KeysetTestHelper:
+ def __init__(self, expect):
+ self.n = 0
+ self.expect = expect
+
+ def fn(self, **kwargs):
+ if self.expect[self.n][0] != kwargs:
+ raise Exception("Didn't match %s != %s" % (self.expect[self.n][0], kwargs))
+ return self
+
+ def execute(self, num_retries):
+ self.n += 1
+ return self.expect[self.n-1][1]
+
+class KeysetListAllTestCase(unittest.TestCase):
+ def test_empty(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [])
+
+ def test_oneitem(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", "=", "1"], ["uuid", ">", "1"]]},
+ {"items": []}
+ ],[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">", "1"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}])
+
+ def test_onepage2(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}])
+
+ def test_onepage3(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "3", "uuid": "3"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "3"], ["uuid", "!=", "3"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "3", "uuid": "3"}])
+
+
+ def test_twopage(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"]]},
+ {"items": [{"created_at": "3", "uuid": "3"}, {"created_at": "4", "uuid": "4"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "4"], ["uuid", "!=", "4"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"},
+ {"created_at": "2", "uuid": "2"},
+ {"created_at": "3", "uuid": "3"},
+ {"created_at": "4", "uuid": "4"}
+ ])
+
+ def test_repeated_key(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "2", "uuid": "3"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "3"]]},
+ {"items": [{"created_at": "2", "uuid": "2"}, {"created_at": "2", "uuid": "4"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", "=", "2"], ["uuid", ">", "4"]]},
+ {"items": []}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">", "2"]]},
+ {"items": [{"created_at": "3", "uuid": "5"}, {"created_at": "4", "uuid": "6"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "4"], ["uuid", "!=", "6"]]},
+ {"items": []}
+ ],
+ ])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"},
+ {"created_at": "2", "uuid": "2"},
+ {"created_at": "2", "uuid": "3"},
+ {"created_at": "2", "uuid": "4"},
+ {"created_at": "3", "uuid": "5"},
+ {"created_at": "4", "uuid": "6"}
+ ])
+
+ def test_onepage_withfilter(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["foo", ">", "bar"]]},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"], ["foo", ">", "bar"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn, filters=[["foo", ">", "bar"]]))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}])
+
+
+ def test_onepage_desc(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "2", "uuid": "2"}, {"created_at": "1", "uuid": "1"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid asc"], "filters": [["created_at", "<=", "1"], ["uuid", "!=", "1"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn, ascending=False))
+ self.assertEqual(ls, [{"created_at": "2", "uuid": "2"}, {"created_at": "1", "uuid": "1"}])
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list