[ARVADOS] updated: 1.3.0-821-g50ab29f0b

Git user git at public.curoverse.com
Mon Apr 29 19:40:59 UTC 2019


Summary of changes:
 sdk/cwl/arvados_cwl/__init__.py                    |  7 ++-
 ...{arv-cwl-schema.yml => arv-cwl-schema-v1.0.yml} |  0
 ...{arv-cwl-schema.yml => arv-cwl-schema-v1.1.yml} | 56 ----------------------
 sdk/cwl/arvados_cwl/arvcontainer.py                |  3 ++
 sdk/cwl/arvados_cwl/arvjob.py                      |  7 ++-
 sdk/cwl/arvados_cwl/executor.py                    | 46 ++++++++++--------
 sdk/cwl/arvados_cwl/runner.py                      |  5 +-
 sdk/cwl/setup.py                                   |  4 +-
 sdk/cwl/tests/test_container.py                    |  2 +-
 9 files changed, 46 insertions(+), 84 deletions(-)
 copy sdk/cwl/arvados_cwl/{arv-cwl-schema.yml => arv-cwl-schema-v1.0.yml} (100%)
 rename sdk/cwl/arvados_cwl/{arv-cwl-schema.yml => arv-cwl-schema-v1.1.yml} (80%)

       via  50ab29f0b8904b1305bcca5b17aa8f5a00f19e76 (commit)
       via  ef0c28eca8c82519c65bca9b528970ba282c6b75 (commit)
       via  ad0a382fbe527cbf35684efa2ebf523507d8ebc2 (commit)
       via  b28e6864b07f4df94cfdb115cfc9b2a0d676d1ba (commit)
       via  71e1650c2e04a5f1c2fcbb2d13da475392996e31 (commit)
      from  f3edcb7d50323b35f3a162b897aaa87030aec424 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 50ab29f0b8904b1305bcca5b17aa8f5a00f19e76
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Mon Apr 29 15:40:24 2019 -0400

    15028: Propagate --enable-dev and handle blank node ids
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 437640ac6..b0dff491f 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -485,6 +485,9 @@ class RunnerContainer(Runner):
         if self.arvrunner.project_uuid:
             command.append("--project-uuid="+self.arvrunner.project_uuid)
 
+        if self.enable_dev:
+            command.append("--enable-dev")
+
         command.extend([workflowpath, "/var/lib/cwl/cwl.input.json"])
 
         container_req["command"] = command
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index badf58938..9b9c2f161 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -365,7 +365,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
 
                 while keys:
                     page = keys[:pageSize]
-                    keys = keys[pageSize:]
                     try:
                         proc_states = table.list(filters=[["uuid", "in", page]]).execute(num_retries=self.num_retries)
                     except Exception:
@@ -381,6 +380,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                                 "new_attributes": p
                             }
                         })
+                    keys = keys[pageSize:]
+
                 finish_poll = time.time()
                 remain_wait = self.poll_interval - (finish_poll - begin_poll)
         except:
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 04fd1c443..ed6bcd008 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -131,7 +131,7 @@ def upload_dependencies(arvrunner, name, document_loader,
         loadref_fields = set(("$import",))
 
     scanobj = workflowobj
-    if "id" in workflowobj:
+    if "id" in workflowobj and not workflowobj["id"].startswith("_:"):
         # Need raw file content (before preprocessing) to ensure
         # that external references in $include and $mixin are captured.
         scanobj = loadref("", workflowobj["id"])
@@ -479,6 +479,7 @@ class Runner(Process):
         self.intermediate_output_ttl = intermediate_output_ttl
         self.priority = priority
         self.secret_store = secret_store
+        self.enable_dev = loadingContext.enable_dev
 
         self.submit_runner_cores = 1
         self.submit_runner_ram = 1024  # defaut 1 GiB

commit ef0c28eca8c82519c65bca9b528970ba282c6b75
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Fri Apr 26 14:53:20 2019 -0400

    15028: Fix discover secondaryFiles after internal v1.1 update
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index c5fd7e3cf..badf58938 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -550,6 +550,10 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         if not runtimeContext.name:
             runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
 
+        # Upload local file references in the job order.
+        job_order = upload_job_order(self, "%s input" % runtimeContext.name,
+                                     tool, job_order)
+
         submitting = (runtimeContext.update_workflow or
                       runtimeContext.create_workflow or
                       (runtimeContext.submit and not
@@ -579,10 +583,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         loadingContext.metadata = tool.metadata
         tool = load_tool(tool.tool, loadingContext)
 
-        # Upload local file references in the job order.
-        job_order = upload_job_order(self, "%s input" % runtimeContext.name,
-                                     tool, job_order)
-
         existing_uuid = runtimeContext.update_workflow
         if existing_uuid or runtimeContext.create_workflow:
             # Create a pipeline template or workflow record and exit.
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 3235f4763..04fd1c443 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -79,7 +79,7 @@ def find_defaults(d, op):
 def setSecondary(t, fileobj, discovered):
     if isinstance(fileobj, dict) and fileobj.get("class") == "File":
         if "secondaryFiles" not in fileobj:
-            fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf), "class": "File"} for sf in t["secondaryFiles"]])
+            fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf["pattern"]), "class": "File"} for sf in t["secondaryFiles"]])
             if discovered is not None:
                 discovered[fileobj["location"]] = fileobj["secondaryFiles"]
     elif isinstance(fileobj, list):

commit ad0a382fbe527cbf35684efa2ebf523507d8ebc2
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Fri Apr 26 14:08:41 2019 -0400

    15028: Bump cwltool
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 37682bc9c..2b273017e 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20190425212529',
+          'cwltool==1.0.20190426175817',
           'schema-salad==4.1.20190305210046',
           'typing >= 3.6.4',
           'ruamel.yaml >=0.15.54, <=0.15.77',

commit b28e6864b07f4df94cfdb115cfc9b2a0d676d1ba
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Fri Apr 26 14:07:05 2019 -0400

    15028: Set the correct loading context to reload tools
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 768783df9..c5fd7e3cf 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -574,6 +574,9 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         # ArvadosCommandTool) because tool document may have been
         # updated by upload_workflow_deps in ways that modify
         # inheritance of hints or requirements.
+        loadingContext.loader = tool.doc_loader
+        loadingContext.avsc_names = tool.doc_schema
+        loadingContext.metadata = tool.metadata
         tool = load_tool(tool.tool, loadingContext)
 
         # Upload local file references in the job order.
@@ -647,8 +650,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         if runtimeContext.submit:
             # Submit a runner job to run the workflow for us.
             if self.work_api == "containers":
-                loadingContext.loader = tool.doc_loader
-                loadingContext.avsc_names = tool.doc_schema
                 if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and (not runtimeContext.always_submit_runner):
                     runtimeContext.runnerjob = tool.tool["id"]
                 else:

commit 71e1650c2e04a5f1c2fcbb2d13da475392996e31
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Apr 25 17:42:49 2019 -0400

    15028: Instead of suppressing update, reload old version for submit
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 95711762c..def720381 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -219,8 +219,11 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
 def add_arv_hints():
     cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*")
     cwltool.command_line_tool.ACCEPTLIST_RE = cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE
-    res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema.yml')
-    use_custom_schema("v1.0", "http://arvados.org/cwl", res.read())
+    res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.0.yml')
+    res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.1.yml')
+    customschema = res.read()
+    use_custom_schema("v1.0", "http://arvados.org/cwl", customschema)
+    use_custom_schema("v1.1.0-dev1", "http://arvados.org/cwl", customschema)
     res.close()
     cwltool.process.supportedProcessRequirements.extend([
         "http://arvados.org/cwl#RunInSingleContainer",
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
similarity index 100%
copy from sdk/cwl/arvados_cwl/arv-cwl-schema.yml
copy to sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
similarity index 80%
rename from sdk/cwl/arvados_cwl/arv-cwl-schema.yml
rename to sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index dce1bd4d0..b9b9e6165 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -9,24 +9,6 @@ $namespaces:
 $graph:
 - $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
 
-- name: cwltool:LoadListingRequirement
-  type: record
-  extends: cwl:ProcessRequirement
-  inVocab: false
-  fields:
-    class:
-      type: string
-      doc: "Always 'LoadListingRequirement'"
-      jsonldPredicate:
-        "_id": "@type"
-        "_type": "@vocab"
-    loadListing:
-      type:
-        - "null"
-        - type: enum
-          name: LoadListingEnum
-          symbols: [no_listing, shallow_listing, deep_listing]
-
 - name: cwltool:Secrets
   type: record
   inVocab: false
@@ -47,28 +29,6 @@ $graph:
         "_type": "@id"
         refScope: 0
 
-- name: cwltool:TimeLimit
-  type: record
-  inVocab: false
-  extends: cwl:ProcessRequirement
-  doc: |
-    Set an upper limit on the execution time of a CommandLineTool or
-    ExpressionTool.  A tool execution which exceeds the time limit may
-    be preemptively terminated and considered failed.  May also be
-    used by batch systems to make scheduling decisions.
-  fields:
-    - name: class
-      type: string
-      doc: "Always 'TimeLimit'"
-      jsonldPredicate:
-        "_id": "@type"
-        "_type": "@vocab"
-    - name: timelimit
-      type: [long, string]
-      doc: |
-        The time limit, in seconds.  A time limit of zero means no
-        time limit.  Negative time limits are an error.
-
 - name: RunInSingleContainer
   type: record
   extends: cwl:ProcessRequirement
@@ -196,22 +156,6 @@ $graph:
         recommended minimum value for TTL is the expected duration of the
         entire the workflow.
 
-- name: ReuseRequirement
-  type: record
-  extends: cwl:ProcessRequirement
-  inVocab: false
-  doc: |
-    Enable/disable work reuse for current process.  Default true (work reuse enabled).
-  fields:
-    - name: class
-      type: string
-      doc: "Always 'arv:ReuseRequirement'"
-      jsonldPredicate:
-        _id: "@type"
-        _type: "@vocab"
-    - name: enableReuse
-      type: boolean
-
 - name: WorkflowRunnerResources
   type: record
   extends: cwl:ProcessRequirement
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index ab2078e15..11efc0c1c 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -236,8 +236,11 @@ class ArvadosJob(JobBase):
         try:
             if record["state"] == "Complete":
                 processStatus = "success"
+                # we don't have the real exit code so fake it.
+                record["exit_code"] = 0
             else:
                 processStatus = "permanentFail"
+                record["exit_code"] = 1
 
             outputs = {}
             try:
@@ -272,8 +275,8 @@ class ArvadosJob(JobBase):
                         outputs = done.done(self, record, dirs["tmpdir"],
                                             dirs["outdir"], dirs["keep"])
             except WorkflowException as e:
-                # Only include a stack trace if in debug mode. 
-                # This is most likely a user workflow error and a stack trace may obfuscate more useful output. 
+                # Only include a stack trace if in debug mode.
+                # This is most likely a user workflow error and a stack trace may obfuscate more useful output.
                 logger.error("%s unable to collect output from %s:\n%s",
                              self.arvrunner.label(self), record["output"], e, exc_info=(e if self.arvrunner.debug else False))
                 processStatus = "permanentFail"
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 20cff6060..768783df9 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -188,7 +188,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         self.loadingContext.fetcher_constructor = self.fetcher_constructor
         self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
         self.loadingContext.construct_tool_object = self.arv_make_tool
-        self.loadingContext.do_update = False
 
         # Add a custom logging handler to the root logger for runtime status reporting
         # if running inside a container
@@ -551,21 +550,31 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         if not runtimeContext.name:
             runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
 
-        # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
-        # Also uploads docker images.
-        merged_map = upload_workflow_deps(self, tool)
+        submitting = (runtimeContext.update_workflow or
+                      runtimeContext.create_workflow or
+                      (runtimeContext.submit and not
+                       (tool.tool["class"] == "CommandLineTool" and
+                        runtimeContext.wait and
+                        not runtimeContext.always_submit_runner)))
 
-        # Reload tool object which may have been updated by
-        # upload_workflow_deps
-        # Don't validate this time because it will just print redundant errors.
         loadingContext = self.loadingContext.copy()
-        loadingContext.loader = tool.doc_loader
-        loadingContext.avsc_names = tool.doc_schema
-        loadingContext.metadata = tool.metadata
         loadingContext.do_validate = False
+        loadingContext.do_update = False
+        if submitting:
+            # Document may have been auto-updated. Reload the original
+            # document with updating disabled because we want to
+            # submit the original document, not the auto-updated one.
+            tool = load_tool(tool.tool["id"], loadingContext)
+
+        # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
+        # Also uploads docker images.
+        merged_map = upload_workflow_deps(self, tool)
 
-        tool = self.arv_make_tool(tool.doc_loader.idx[tool.tool["id"]],
-                                  loadingContext)
+        # Recreate process object (ArvadosWorkflow or
+        # ArvadosCommandTool) because tool document may have been
+        # updated by upload_workflow_deps in ways that modify
+        # inheritance of hints or requirements.
+        tool = load_tool(tool.tool, loadingContext)
 
         # Upload local file references in the job order.
         job_order = upload_job_order(self, "%s input" % runtimeContext.name,
@@ -638,6 +647,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         if runtimeContext.submit:
             # Submit a runner job to run the workflow for us.
             if self.work_api == "containers":
+                loadingContext.loader = tool.doc_loader
+                loadingContext.avsc_names = tool.doc_schema
                 if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and (not runtimeContext.always_submit_runner):
                     runtimeContext.runnerjob = tool.tool["id"]
                 else:
@@ -673,11 +684,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                     "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
             logger.info("Pipeline instance %s", self.pipeline["uuid"])
 
-        if not isinstance(tool, Runner):
-            loadingContext.do_update = True
-            tool = load_tool(tool.doc_loader.idx[tool.tool["id"]],
-                             loadingContext)
-
         if runtimeContext.cwl_runner_job is not None:
             self.uuid = runtimeContext.cwl_runner_job.get('uuid')
 
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 79c5fc209..37682bc9c 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -25,7 +25,7 @@ setup(name='arvados-cwl-runner',
       download_url="https://github.com/curoverse/arvados.git",
       license='Apache 2.0',
       packages=find_packages(),
-      package_data={'arvados_cwl': ['arv-cwl-schema.yml']},
+      package_data={'arvados_cwl': ['arv-cwl-schema-v1.0.yml', 'arv-cwl-schema-v1.1.yml']},
       scripts=[
           'bin/cwl-runner',
           'bin/arvados-cwl-runner',
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20190423203253',
+          'cwltool==1.0.20190425212529',
           'schema-salad==4.1.20190305210046',
           'typing >= 3.6.4',
           'ruamel.yaml >=0.15.54, <=0.15.77',
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 2ddd06535..1f169cf23 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -513,7 +513,7 @@ class TestContainer(unittest.TestCase):
         self.assertFalse(api.collections().create.called)
         self.assertFalse(runner.runtime_status_error.called)
 
-        arvjob.collect_outputs.assert_called_with("keep:abc+123")
+        arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list