[ARVADOS] updated: 2.1.0-2389-gbea50608b

Git user git at public.arvados.org
Wed May 4 21:11:28 UTC 2022


Summary of changes:
 sdk/cwl/arvados_cwl/executor.py                    |   6 +-
 .../example3.cwl => 19070-copy-deps.cwl}           |  13 +-
 sdk/cwl/tests/arvados-tests.sh                     |   9 ++
 sdk/cwl/tests/test_copy_deps.py                    | 142 +++++++++++++++++++++
 sdk/python/tests/test_arv_keepdocker.py            |   8 +-
 5 files changed, 167 insertions(+), 11 deletions(-)
 copy sdk/cwl/tests/{secondaryFiles/example3.cwl => 19070-copy-deps.cwl} (62%)
 create mode 100644 sdk/cwl/tests/test_copy_deps.py

       via  bea50608be40f5ecf218fa5be75213579ad95eca (commit)
      from  b525a0cd35eba338abf78c5bb2b392c2e580d573 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit bea50608be40f5ecf218fa5be75213579ad95eca
Author: Peter Amstutz <peter.amstutz at curii.com>
Date:   Wed May 4 16:57:01 2022 -0400

    19070: Add integration test for copying dependencies
    
    Fix python tests
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index ed57c6dae..eace9f449 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -544,10 +544,10 @@ The 'jobs' API is no longer supported.
         if not runtimeContext.name:
             runtimeContext.name = self.name = updated_tool.tool.get("label") or updated_tool.metadata.get("label") or os.path.basename(updated_tool.tool["id"])
 
-        if runtimeContext.copy_deps is None and (runtimeContext.create_workflow or runtimeContext.update_workflow):
-            runtimeContext.copy_deps = True
+        if self.runtimeContext.copy_deps is None and (runtimeContext.create_workflow or runtimeContext.update_workflow):
+            self.runtimeContext.copy_deps = True
 
-        if runtimeContext.update_workflow and self.project_uuid is None:
+        if self.runtimeContext.update_workflow and self.project_uuid is None:
             # If we are updating a workflow, make sure anything that
             # gets uploaded goes into the same parent project, unless
             # an alternate --project-uuid was provided.
diff --git a/sdk/cwl/tests/19070-copy-deps.cwl b/sdk/cwl/tests/19070-copy-deps.cwl
new file mode 100644
index 000000000..b0d61700e
--- /dev/null
+++ b/sdk/cwl/tests/19070-copy-deps.cwl
@@ -0,0 +1,17 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: echo
+inputs:
+  message:
+    type: File
+    inputBinding:
+      position: 1
+    default:
+      class: File
+      location: keep:d7514270f356df848477718d58308cc4+94/b
+
+outputs: []
diff --git a/sdk/cwl/tests/arvados-tests.sh b/sdk/cwl/tests/arvados-tests.sh
index 9cb5234cf..1bbaa505e 100755
--- a/sdk/cwl/tests/arvados-tests.sh
+++ b/sdk/cwl/tests/arvados-tests.sh
@@ -18,6 +18,15 @@ if ! arv-get 20850f01122e860fb878758ac1320877+71 > /dev/null ; then
     arv-put --portable-data-hash samples/sample1_S01_R1_001.fastq.gz
 fi
 
+# Test for #18888
+# This is a standalone test because the bug was observed with this
+# command line and was thought to be due to command line handling.
 arvados-cwl-runner 18888-download_def.cwl --scripts scripts/
 
+# Test for #19070
+# The most effective way to test this seemed to be to write an
+# integration test to check for the expected behavior.
+python test_copy_deps.py
+
+# Run integration tests
 exec cwltest --test arvados-tests.yml --tool arvados-cwl-runner $@ -- --disable-reuse --compute-checksum --api=containers
diff --git a/sdk/cwl/tests/test_copy_deps.py b/sdk/cwl/tests/test_copy_deps.py
new file mode 100644
index 000000000..298a6800f
--- /dev/null
+++ b/sdk/cwl/tests/test_copy_deps.py
@@ -0,0 +1,142 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import subprocess
+
+api = arvados.api()
+
+def check_contents(group, wf_uuid):
+    contents = api.groups().contents(uuid=group["uuid"]).execute()
+    if len(contents["items"]) != 3:
+        raise Exception("Expected 3 items")
+
+    found = False
+    for c in contents["items"]:
+        if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
+            found = True
+    if not found:
+        raise Exception("Couldn't find workflow")
+
+    found = False
+    for c in contents["items"]:
+        if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
+            found = True
+    if not found:
+        raise Exception("Couldn't find collection dependency")
+
+    found = False
+    for c in contents["items"]:
+        if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
+            found = True
+    if not found:
+        raise Exception("Couldn't find jobs image dependency")
+
+
+def test_create():
+    group = api.groups().create(body={"group": {"name": "test-19070-project-1", "group_class": "project"}}, ensure_unique_name=True).execute()
+    try:
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        if len(contents["items"]) != 0:
+            raise Exception("Expected 0 items")
+
+        # Create workflow, by default should also copy dependencies
+        wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"])
+        wf_uuid = wf_uuid.decode("utf-8").strip()
+        check_contents(group, wf_uuid)
+    finally:
+        api.groups().delete(uuid=group["uuid"]).execute()
+
+
+def test_update():
+    group = api.groups().create(body={"group": {"name": "test-19070-project-2", "group_class": "project"}}, ensure_unique_name=True).execute()
+    try:
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        if len(contents["items"]) != 0:
+            raise Exception("Expected 0 items")
+
+        # Create workflow, but with --no-copy-deps it shouldn't copy anything
+        wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--no-copy-deps", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"])
+        wf_uuid = wf_uuid.decode("utf-8").strip()
+
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        if len(contents["items"]) != 1:
+            raise Exception("Expected 1 items")
+
+        found = False
+        for c in contents["items"]:
+            if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
+                found = True
+        if not found:
+            raise Exception("Couldn't find workflow")
+
+        # Updating by default will copy missing items
+        wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--update-workflow", wf_uuid, "19070-copy-deps.cwl"])
+        wf_uuid = wf_uuid.decode("utf-8").strip()
+        check_contents(group, wf_uuid)
+
+    finally:
+        api.groups().delete(uuid=group["uuid"]).execute()
+
+
+def test_execute():
+    group = api.groups().create(body={"group": {"name": "test-19070-project-3", "group_class": "project"}}, ensure_unique_name=True).execute()
+    try:
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        if len(contents["items"]) != 0:
+            raise Exception("Expected 0 items")
+
+        # Execute workflow, shouldn't copy anything.
+        wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"])
+        wf_uuid = wf_uuid.decode("utf-8").strip()
+
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        # container request
+        # final output collection
+        # container log
+        # step output collection
+        # container request log
+        if len(contents["items"]) != 5:
+            raise Exception("Expected 5 items")
+
+        found = False
+        for c in contents["items"]:
+            if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
+                found = True
+        if found:
+            raise Exception("Didn't expect to find collection dependency")
+
+        found = False
+        for c in contents["items"]:
+            if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
+                found = True
+        if found:
+            raise Exception("Didn't expect to find jobs image dependency")
+
+        # Execute workflow with --copy-deps
+        wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--project-uuid", group["uuid"], "--copy-deps", "19070-copy-deps.cwl"])
+        wf_uuid = wf_uuid.decode("utf-8").strip()
+
+        contents = api.groups().contents(uuid=group["uuid"]).execute()
+        found = False
+        for c in contents["items"]:
+            if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
+                found = True
+        if not found:
+            raise Exception("Couldn't find collection dependency")
+
+        found = False
+        for c in contents["items"]:
+            if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
+                found = True
+        if not found:
+            raise Exception("Couldn't find jobs image dependency")
+
+    finally:
+        api.groups().delete(uuid=group["uuid"]).execute()
+
+
+test_create()
+test_update()
+test_execute()
diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py
index fd3a69cae..8fbfad437 100644
--- a/sdk/python/tests/test_arv_keepdocker.py
+++ b/sdk/python/tests/test_arv_keepdocker.py
@@ -48,11 +48,13 @@ class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
                 self.run_arv_keepdocker(['--version'], sys.stderr)
         self.assertVersionOutput(out, err)
 
+    @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
+                return_value=[])
     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
                 return_value=['abc123'])
     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
                 return_value='abc123')
-    def test_image_format_compatibility(self, _1, _2):
+    def test_image_format_compatibility(self, _1, _2, _3):
         old_id = hashlib.sha256(b'old').hexdigest()
         new_id = 'sha256:'+hashlib.sha256(b'new').hexdigest()
         for supported, img_id, expect_ok in [
@@ -152,11 +154,13 @@ class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker):
             self.run_arv_keepdocker(['[::1]/repo/img'], sys.stderr)
         find_image_mock.assert_called_with('[::1]/repo/img', 'latest')
 
+    @mock.patch('arvados.commands.keepdocker.list_images_in_arv',
+                return_value=[])
     @mock.patch('arvados.commands.keepdocker.find_image_hashes',
                 return_value=['abc123'])
     @mock.patch('arvados.commands.keepdocker.find_one_image_hash',
                 return_value='abc123')
-    def test_collection_property_update(self, _1, _2):
+    def test_collection_property_update(self, _1, _2, _3):
         image_id = 'sha256:'+hashlib.sha256(b'image').hexdigest()
         fakeDD = arvados.api('v1')._rootDesc
         fakeDD['dockerImageFormats'] = ['v2']

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list