[ARVADOS] created: 1.1.4-248-g7ab351a

Git user git at public.curoverse.com
Wed May 9 14:28:50 EDT 2018


        at  7ab351a4f34e71e4351481eb2a5b41e5d06a52a7 (commit)


commit 7ab351a4f34e71e4351481eb2a5b41e5d06a52a7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Wed May 9 14:28:14 2018 -0400

    12256: Bump cwltool version for workflow packing bugfix.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index c636bb8..48b1903 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20180504184252',
+          'cwltool==1.0.20180508202931',
           'schema-salad==2.7.20180501211602',
           'typing==3.5.3.0',
           'ruamel.yaml >=0.13.7, <0.15',

commit 3b4da80a23136c0cec743ee4cdad9b22b51ac3e0
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Fri May 4 16:44:29 2018 -0400

    12256: Handle default references in RunInSingleContainer
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index ddc270d..8268300 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -24,7 +24,7 @@ class ArvadosCommandTool(CommandLineTool):
     def makePathMapper(self, reffiles, stagedir, **kwargs):
         # type: (List[Any], unicode, **Any) -> PathMapper
         if self.work_api == "containers":
-            return ArvPathMapper(self.arvrunner, reffiles, kwargs["basedir"],
+            return ArvPathMapper(self.arvrunner, reffiles+kwargs.get("extra_reffiles", []), kwargs["basedir"],
                                  "/keep/%s",
                                  "/keep/%s/%s",
                                  **kwargs)
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index bdc2e27..ba16b86 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -205,13 +205,17 @@ class ArvadosWorkflow(Workflow):
                 joborder_keepmount = copy.deepcopy(joborder)
 
                 reffiles = []
-                visit_class(joborder_keepmount, ("File", "Directory"), lambda x: reffiles.append(x))
+                visit_class(joborder_keepmount, ("File", "Directory"), reffiles.append)
+                reffiles2 = []
+                visit_class(packed, ("File", "Directory"), reffiles2.append)
 
-                mapper = ArvPathMapper(self.arvrunner, reffiles, kwargs["basedir"],
+                mapper = ArvPathMapper(self.arvrunner, reffiles+reffiles2, kwargs["basedir"],
                                  "/keep/%s",
                                  "/keep/%s/%s",
                                  **kwargs)
 
+                kwargs["extra_reffiles"] = copy.deepcopy(reffiles2)
+
                 def keepmount(obj):
                     remove_redundant_fields(obj)
                     with SourceLine(obj, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index fe1ef22..3ce08f6 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -217,13 +217,13 @@ def packed_workflow(arvrunner, tool, merged_map):
     packed = pack(tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]),
                   tool.tool["id"], tool.metadata, rewrite_out=rewrites)
 
-    rewrite_to_orig = {}
-    for k,v in rewrites.items():
-        rewrite_to_orig[v] = k
+    rewrite_to_orig = {v: k for k,v in rewrites.items()}
 
     def visit(v, cur_id):
         if isinstance(v, dict):
             if v.get("class") in ("CommandLineTool", "Workflow"):
+                if "id" not in v:
+                    raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
                 cur_id = rewrite_to_orig.get(v["id"], v["id"])
             if "location" in v and not v["location"].startswith("keep:"):
                 v["location"] = merged_map[cur_id].resolved[v["location"]]
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 563015c..c636bb8 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20180502225535',
+          'cwltool==1.0.20180504184252',
           'schema-salad==2.7.20180501211602',
           'typing==3.5.3.0',
           'ruamel.yaml >=0.13.7, <0.15',
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 172e6a9..9106426 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -199,7 +199,7 @@
   output:
     out: null
   tool: wf-defaults/wf3.cwl
-  doc: "Do not accept a directory literal without a basename
+  doc: "Do not accept a directory literal without a basename"
   should_fail: true
 
 - job: null

commit bd3e0a105044c0b61c3b07649c6c284f5f1a641b
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu May 3 12:34:05 2018 -0400

    12256: Add a bunch of test cases.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 1950867..172e6a9 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -182,3 +182,46 @@
   output: {}
   tool: secondaryFiles/example3.cwl
   doc: Discover secondaryFiles on default values
+
+- job: null
+  output:
+    out: null
+  tool: wf-defaults/wf1.cwl
+  doc: "Can have separate default parameters including directory and file inside same directory"
+
+- job: null
+  output:
+    out: null
+  tool: wf-defaults/wf2.cwl
+  doc: "Can have a parameter default value that is a directory literal with a file literal"
+
+- job: null
+  output:
+    out: null
+  tool: wf-defaults/wf3.cwl
+  doc: "Do not accept a directory literal without a basename
+  should_fail: true
+
+- job: null
+  output:
+    out: null
+  tool: wf-defaults/wf4.cwl
+  doc: default in embedded subworkflow
+
+- job: null
+  output:
+    out: null
+  tool: wf-defaults/wf5.cwl
+  doc: default in embedded subworkflow
+
+- job: null
+  output:
+    out: null
+  tool: wf-defaults/wf6.cwl
+  doc: default in RunInSingleContainer step
+
+- job: null
+  output:
+    out: null
+  tool: wf-defaults/wf7.cwl
+  doc: workflow level default in RunInSingleContainer
diff --git a/sdk/cwl/tests/wf-defaults/default-dir1.cwl b/sdk/cwl/tests/wf-defaults/default-dir1.cwl
new file mode 100644
index 0000000..ed09e6e
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir1.cwl
@@ -0,0 +1,15 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+  inp2:
+    type: Directory
+    default:
+      class: Directory
+      location: inp1
+  inp1:
+    type: File
+    default:
+      class: File
+      location: inp1/hello.txt
+outputs: []
+arguments: [echo, $(inputs.inp1), $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir2.cwl b/sdk/cwl/tests/wf-defaults/default-dir2.cwl
new file mode 100644
index 0000000..c826464
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir2.cwl
@@ -0,0 +1,14 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+  inp2:
+    type: Directory
+    default:
+      class: Directory
+      basename: inp2
+      listing:
+        - class: File
+          basename: "hello.txt"
+          contents: "hello world"
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir3.cwl b/sdk/cwl/tests/wf-defaults/default-dir3.cwl
new file mode 100644
index 0000000..ab7b0a4
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir3.cwl
@@ -0,0 +1,12 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+  inp2:
+    type: Directory
+    default:
+      class: Directory
+      listing:
+        - class: File
+          location: "inp1/hello.txt"
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir4.cwl b/sdk/cwl/tests/wf-defaults/default-dir4.cwl
new file mode 100644
index 0000000..cd57ff3
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir4.cwl
@@ -0,0 +1,20 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+steps:
+  step1:
+    in: []
+    out: []
+    run:
+      class: CommandLineTool
+      inputs:
+        inp2:
+          type: Directory
+          default:
+            class: Directory
+            location: inp1
+      outputs: []
+      arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir5.cwl b/sdk/cwl/tests/wf-defaults/default-dir5.cwl
new file mode 100644
index 0000000..d4f667c
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir5.cwl
@@ -0,0 +1,21 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+steps:
+  step1:
+    in: []
+    out: []
+    run:
+      id: stepid
+      class: CommandLineTool
+      inputs:
+        inp2:
+          type: Directory
+          default:
+            class: Directory
+            location: inp1
+      outputs: []
+      arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir6.cwl b/sdk/cwl/tests/wf-defaults/default-dir6.cwl
new file mode 100644
index 0000000..597ea96
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir6.cwl
@@ -0,0 +1,11 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+steps:
+  step1:
+    in: []
+    out: []
+    run: default-dir6a.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir6a.cwl b/sdk/cwl/tests/wf-defaults/default-dir6a.cwl
new file mode 100644
index 0000000..76437a2
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir6a.cwl
@@ -0,0 +1,10 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+  inp2:
+    type: Directory
+    default:
+      class: Directory
+      location: inp1
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir7.cwl b/sdk/cwl/tests/wf-defaults/default-dir7.cwl
new file mode 100644
index 0000000..4e6372b
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir7.cwl
@@ -0,0 +1,17 @@
+cwlVersion: v1.0
+class: Workflow
+inputs:
+  inp2:
+    type: Directory
+    default:
+      class: Directory
+      location: inp1
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+steps:
+  step1:
+    in:
+      inp2: inp2
+    out: []
+    run: default-dir7a.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/default-dir7a.cwl b/sdk/cwl/tests/wf-defaults/default-dir7a.cwl
new file mode 100644
index 0000000..df9009a
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir7a.cwl
@@ -0,0 +1,7 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+  inp2:
+    type: Directory
+outputs: []
+arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/inp1/hello.txt b/sdk/cwl/tests/wf-defaults/inp1/hello.txt
new file mode 100644
index 0000000..e69de29
diff --git a/sdk/cwl/tests/wf-defaults/wf1.cwl b/sdk/cwl/tests/wf-defaults/wf1.cwl
new file mode 100644
index 0000000..45faa89
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf1.cwl
@@ -0,0 +1,9 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+steps:
+  step1:
+    in: []
+    out: []
+    run: default-dir1.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf2.cwl b/sdk/cwl/tests/wf-defaults/wf2.cwl
new file mode 100644
index 0000000..7ba96ee
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf2.cwl
@@ -0,0 +1,9 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+steps:
+  step1:
+    in: []
+    out: []
+    run: default-dir2.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf3.cwl b/sdk/cwl/tests/wf-defaults/wf3.cwl
new file mode 100644
index 0000000..911650d
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf3.cwl
@@ -0,0 +1,9 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+steps:
+  step1:
+    in: []
+    out: []
+    run: default-dir3.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf4.cwl b/sdk/cwl/tests/wf-defaults/wf4.cwl
new file mode 100644
index 0000000..d6e65af
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf4.cwl
@@ -0,0 +1,13 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+requirements:
+  SubworkflowFeatureRequirement: {}
+steps:
+  step1:
+    in: []
+    out: []
+    run: default-dir4.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf5.cwl b/sdk/cwl/tests/wf-defaults/wf5.cwl
new file mode 100644
index 0000000..631af18
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf5.cwl
@@ -0,0 +1,13 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+requirements:
+  SubworkflowFeatureRequirement: {}
+steps:
+  step1:
+    in: []
+    out: []
+    run: default-dir5.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf6.cwl b/sdk/cwl/tests/wf-defaults/wf6.cwl
new file mode 100644
index 0000000..bd26cc1
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf6.cwl
@@ -0,0 +1,15 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+requirements:
+  SubworkflowFeatureRequirement: {}
+steps:
+  step1:
+    requirements:
+      arv:RunInSingleContainer: {}
+    in: []
+    out: []
+    run: default-dir6.cwl
\ No newline at end of file
diff --git a/sdk/cwl/tests/wf-defaults/wf7.cwl b/sdk/cwl/tests/wf-defaults/wf7.cwl
new file mode 100644
index 0000000..ac07b9d
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf7.cwl
@@ -0,0 +1,15 @@
+cwlVersion: v1.0
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+requirements:
+  SubworkflowFeatureRequirement: {}
+steps:
+  step1:
+    requirements:
+      arv:RunInSingleContainer: {}
+    in: []
+    out: []
+    run: default-dir7.cwl
\ No newline at end of file

commit 8eadbd97fba7c45896558d0aac508756ea557be7
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu May 3 10:37:44 2018 -0400

    12256: Fix anonymous directory default inputs with file literals
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 998890a..6fedb12 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -107,6 +107,7 @@ class ArvPathMapper(PathMapper):
         elif obj["location"].startswith("_:") and "contents" in obj:
             with c.open(path + "/" + obj["basename"], "w") as f:
                 f.write(obj["contents"].encode("utf-8"))
+            remap.append((obj["location"], path + "/" + obj["basename"]))
         else:
             raise SourceLine(obj, "location", WorkflowException).makeError("Don't know what to do with '%s'" % obj["location"])
 

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list