[ARVADOS] created: ecf3cf1ec8544cffacc620919e35404e086b931a

Git user git at public.curoverse.com
Mon Sep 11 08:11:44 EDT 2017


        at  ecf3cf1ec8544cffacc620919e35404e086b931a (commit)


commit ecf3cf1ec8544cffacc620919e35404e086b931a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Aug 29 12:56:55 2017 -0400

    12186: Filter nameroot from runner-submitted input object.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 769a63b..78086ed 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -14,14 +14,14 @@ import ruamel.yaml as yaml
 
 from cwltool.errors import WorkflowException
 from cwltool.process import get_feature, UnsupportedRequirement, shortname
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 from cwltool.utils import aslist
 
 import arvados.collection
 
 from .arvdocker import arv_docker_get_image
 from . import done
-from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location
+from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location, remove_redundant_fields
 from .fsaccess import CollectionFetcher
 from .pathmapper import NoFollowPathMapper, trim_listing
 from .perf import Perf
@@ -286,8 +286,8 @@ class RunnerContainer(Runner):
         """
 
         adjustDirObjs(self.job_order, trim_listing)
-        adjustFileObjs(self.job_order, trim_anonymous_location)
-        adjustDirObjs(self.job_order, trim_anonymous_location)
+        visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location)
+        visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields)
 
         container_req = {
             "owner_uuid": self.arvrunner.project_uuid,
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 64cd2aa..d104d56 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -13,7 +13,7 @@ from cwltool.errors import WorkflowException
 from cwltool.draft2tool import revmap_file, CommandLineTool
 from cwltool.load_tool import fetch_document
 from cwltool.builder import Builder
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 
 from schema_salad.sourceline import SourceLine
 
@@ -23,7 +23,7 @@ import arvados.collection
 from arvados.errors import ApiError
 
 from .arvdocker import arv_docker_get_image
-from .runner import Runner, arvados_jobs_image, packed_workflow, upload_workflow_collection, trim_anonymous_location
+from .runner import Runner, arvados_jobs_image, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields
 from .pathmapper import VwdPathMapper, trim_listing
 from .perf import Perf
 from . import done
@@ -282,8 +282,8 @@ class RunnerJob(Runner):
             self.job_order["cwl:tool"] = "%s/workflow.cwl#main" % wf_pdh
 
         adjustDirObjs(self.job_order, trim_listing)
-        adjustFileObjs(self.job_order, trim_anonymous_location)
-        adjustDirObjs(self.job_order, trim_anonymous_location)
+        visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location)
+        visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields)
 
         if self.output_name:
             self.job_order["arv:output_name"] = self.output_name
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 687bead..20cb467 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -17,7 +17,7 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 
 import ruamel.yaml as yaml
 
-from .runner import upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location
+from .runner import upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields
 from .pathmapper import trim_listing
 from .arvtool import ArvadosCommandTool
 from .perf import Perf
@@ -113,9 +113,7 @@ class ArvadosWorkflow(Workflow):
                 joborder_keepmount = copy.deepcopy(joborder)
 
                 def keepmount(obj):
-                    for field in ("path", "nameext", "nameroot", "dirname"):
-                        if field in obj:
-                            del obj[field]
+                    remove_redundant_fields(obj)
                     with SourceLine(obj, None, WorkflowException):
                         if "location" not in obj:
                             raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index bb4fac2..3949709 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -46,6 +46,11 @@ def trim_anonymous_location(obj):
     if obj.get("location", "").startswith("_:"):
         del obj["location"]
 
+def remove_redundant_fields(obj):
+    for field in ("path", "nameext", "nameroot", "dirname"):
+        if field in obj:
+            del obj[field]
+
 def find_defaults(d, op):
     if isinstance(d, list):
         for i in d:
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 49545a8..0314720 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -157,8 +157,6 @@ def stubs(func):
                             'basename': 'blorp.txt',
                             'class': 'File',
                             'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
-                            "nameext": ".txt",
-                            "nameroot": "blorp",
                             "size": 16
                         }},
                         'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
@@ -166,9 +164,7 @@ def stubs(func):
                                   {
                                       'basename': 'renamed.txt',
                                       'class': 'File', 'location':
-                                      'keep:99999999999999999999999999999998+99/file1.txt',
-                                      "nameext": ".txt",
-                                      "nameroot": "renamed"
+                                      'keep:99999999999999999999999999999998+99/file1.txt'
                                   }
                               ]}},
                         'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
@@ -223,16 +219,12 @@ def stubs(func):
                             'basename': u'blorp.txt',
                             'class': 'File',
                             'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
-                            "nameext": ".txt",
-                            "nameroot": "blorp",
                             "size": 16
                         },
                         'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
                             {'basename': 'renamed.txt',
                              'class': 'File',
-                             'location': 'keep:99999999999999999999999999999998+99/file1.txt',
-                             "nameext": ".txt",
-                             "nameroot": "renamed"
+                             'location': 'keep:99999999999999999999999999999998+99/file1.txt'
                             }
                         ]}
                     },

commit 45054326217ef62eeef9bcdbd0ed429c55650609
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Aug 29 12:48:59 2017 -0400

    12186: Filter nameroot/nameext from RunInSingleContainer and update test.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 1361579..687bead 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -13,7 +13,7 @@ from cwltool.pack import pack
 from cwltool.load_tool import fetch_document
 from cwltool.process import shortname
 from cwltool.workflow import Workflow, WorkflowException
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 
 import ruamel.yaml as yaml
 
@@ -113,6 +113,9 @@ class ArvadosWorkflow(Workflow):
                 joborder_keepmount = copy.deepcopy(joborder)
 
                 def keepmount(obj):
+                    for field in ("path", "nameext", "nameroot", "dirname"):
+                        if field in obj:
+                            del obj[field]
                     with SourceLine(obj, None, WorkflowException):
                         if "location" not in obj:
                             raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
@@ -126,8 +129,7 @@ class ArvadosWorkflow(Workflow):
                         else:
                             raise WorkflowException("Location is not a keep reference or a literal: '%s'" % obj["location"])
 
-                adjustFileObjs(joborder_keepmount, keepmount)
-                adjustDirObjs(joborder_keepmount, keepmount)
+                visit_class(joborder_keepmount, ("File", "Directory"), keepmount)
 
                 if self.wf_pdh is None:
                     adjustFileObjs(packed, keepmount)
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 5d140ba..faac1f7 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -306,9 +306,10 @@ class TestJob(unittest.TestCase):
 class TestWorkflow(unittest.TestCase):
     # The test passes no builder.resources
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+    @mock.patch("arvados.collection.CollectionReader")
     @mock.patch("arvados.collection.Collection")
     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
-    def test_run(self, list_images_in_arv, mockcollection):
+    def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
         arvados_cwl.add_arv_hints()
 
         api = mock.MagicMock()
@@ -319,19 +320,25 @@ class TestWorkflow(unittest.TestCase):
 
         list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
         runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+        runner.api.collections().list().execute.return_vaulue = {"items": [{"portable_data_hash": "99999999999999999999999999999993+99"}]}
 
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.num_retries = 0
         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 
+        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                         collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+        document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=api, fs_access=make_fs_access(""))
+        document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
+        document_loader.fetch_text = document_loader.fetcher.fetch_text
+        document_loader.check_exists = document_loader.fetcher.check_exists
+
         tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
         metadata["cwlVersion"] = tool["cwlVersion"]
 
         mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
 
-        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
-                                         collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
                                               basedir="", make_fs_access=make_fs_access, loader=document_loader,
                                               makeTool=runner.arv_make_tool, metadata=metadata)
@@ -375,6 +382,11 @@ class TestWorkflow(unittest.TestCase):
         mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
         mockcollection().open().__enter__().write.assert_has_calls([mock.call(
 '''{
+  "fileblub": {
+    "basename": "token.txt",
+    "class": "File",
+    "location": "/keep/99999999999999999999999999999999+118/token.txt"
+  },
   "sleeptime": 5
 }''')])
 
diff --git a/sdk/cwl/tests/wf/scatter2.cwl b/sdk/cwl/tests/wf/scatter2.cwl
index e6c002d..79c3854 100644
--- a/sdk/cwl/tests/wf/scatter2.cwl
+++ b/sdk/cwl/tests/wf/scatter2.cwl
@@ -10,6 +10,11 @@ inputs:
   sleeptime:
     type: int[]
     default: [5]
+  fileblub:
+    type: File
+    default:
+      class: File
+      location: keep:99999999999999999999999999999999+118/token.txt
 outputs:
   out:
     type: string[]
@@ -23,6 +28,7 @@ steps:
   scatterstep:
     in:
       sleeptime: sleeptime
+      fileblub: fileblub
     out: [out]
     scatter: sleeptime
     hints:
@@ -32,6 +38,7 @@ steps:
       id: mysub
       inputs:
         sleeptime: int
+        fileblub: File
       outputs:
         out:
           type: string
diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl
index 2a9cea5..2af1155 100644
--- a/sdk/cwl/tests/wf/scatter2_subwf.cwl
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@ -11,6 +11,10 @@
       "id": "#main",
       "inputs": [
         {
+          "id": "#main/fileblub",
+          "type": "File"
+        },
+        {
           "id": "#main/sleeptime",
           "type": "int"
         }

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list