[ARVADOS] updated: 60d5347a73a24638b57715c5f5d9c56d84a9e57e
Git user
git at public.curoverse.com
Tue Sep 27 15:57:47 EDT 2016
Summary of changes:
sdk/cwl/arvados_cwl/arvworkflow.py | 7 ++++---
sdk/cwl/arvados_cwl/runner.py | 6 ++++--
sdk/cwl/tests/submit_test_job.json | 9 +++++++++
sdk/cwl/tests/test_submit.py | 20 +++++++++++++++++---
sdk/cwl/tests/wf/expect_packed.cwl | 7 +++++++
sdk/cwl/tests/wf/submit_wf.cwl | 2 ++
6 files changed, 43 insertions(+), 8 deletions(-)
via 60d5347a73a24638b57715c5f5d9c56d84a9e57e (commit)
via 04f0197d84cbde94a487ab695357befa2f47a7de (commit)
from e378cc27decd91bf72309946b84c7346a75be7c1 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 60d5347a73a24638b57715c5f5d9c56d84a9e57e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Sep 27 15:57:40 2016 -0400
10129: Ensure that directory literals get preserved when trimming keep directory listings.
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 5976081..3bb0a34 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -85,12 +85,13 @@ class ArvadosWorkflow(Workflow):
def keepmount(obj):
if obj["location"].startswith("keep:"):
obj["location"] = "/keep/" + obj["location"][5:]
+ if "listing" in obj:
+ del obj["listing"]
elif obj["location"].startswith("_:"):
- pass
+ del obj["location"]
else:
raise WorkflowException("Location is not a keep reference or a literal: '%s'" % obj["location"])
- if "listing" in obj:
- del obj["listing"]
+
adjustFileObjs(joborder_keepmount, keepmount)
adjustDirObjs(joborder_keepmount, keepmount)
adjustFileObjs(packed, keepmount)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 4e91d66..aafe7c3 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -26,6 +26,8 @@ cwltool.draft2tool.ACCEPTLIST_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
def del_listing(obj):
if obj.get("location", "").startswith("keep:") and "listing" in obj:
del obj["listing"]
+ if obj.get("location", "").startswith("_:"):
+ del obj["location"]
def upload_dependencies(arvrunner, name, document_loader,
workflowobj, uri, loadref_run):
@@ -84,7 +86,7 @@ def upload_dependencies(arvrunner, name, document_loader,
name=name)
def setloc(p):
- if not p["location"].startswith("_:") and not p["location"].startswith("keep:"):
+ if "location" in p and (not p["location"].startswith("_:")) and (not p["location"].startswith("keep:")):
p["location"] = mapper.mapper(p["location"]).resolved
adjustFileObjs(workflowobj, setloc)
adjustDirObjs(workflowobj, setloc)
diff --git a/sdk/cwl/tests/submit_test_job.json b/sdk/cwl/tests/submit_test_job.json
index 02d61fa..49d5944 100644
--- a/sdk/cwl/tests/submit_test_job.json
+++ b/sdk/cwl/tests/submit_test_job.json
@@ -10,5 +10,14 @@
"class": "File",
"location": "keep:99999999999999999999999999999998+99/file1.txt"
}]
+ },
+ "z": {
+ "class": "Directory",
+ "basename": "anonymous",
+ "listing": [{
+ "basename": "renamed.txt",
+ "class": "File",
+ "location": "keep:99999999999999999999999999999998+99/file1.txt"
+ }]
}
}
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index f57ff32..eb71992 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -93,6 +93,15 @@ def stubs(func):
'location': 'keep:99999999999999999999999999999998+99',
'class': 'Directory'
},
+ 'z': {
+ 'basename': 'anonymous',
+ "listing": [{
+ "basename": "renamed.txt",
+ "class": "File",
+ "location": "keep:99999999999999999999999999999998+99/file1.txt"
+ }],
+ 'class': 'Directory'
+ },
'cwl:tool':
'99999999999999999999999999999991+99/wf/submit_wf.cwl'
},
@@ -117,7 +126,7 @@ def stubs(func):
'kind': 'file'
},
'/var/lib/cwl/job/cwl.input.json': {
- 'portable_data_hash': '606be75b6e4f811a2f282d7fac867043+60/cwl.input.json',
+ 'portable_data_hash': 'd20d7cddd1984f105dd3702c7f125afb+60/cwl.input.json',
'kind': 'collection'
}
},
@@ -160,7 +169,7 @@ class TestSubmit(unittest.TestCase):
'manifest_text':
'./tool d51232d96b6116d964a69bfb7e0c73bf+450 '
'0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
- '0f8864f292e901019c43fdabacd62c3e+383 0:383:submit_wf.cwl\n',
+ 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'name': 'submit_wf.cwl',
}, ensure_unique_name=True),
@@ -220,7 +229,7 @@ class TestSubmit(unittest.TestCase):
'manifest_text':
'./tool d51232d96b6116d964a69bfb7e0c73bf+450 '
'0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
- '0f8864f292e901019c43fdabacd62c3e+383 0:383:submit_wf.cwl\n',
+ 'cc2ffb940e60adf1b2b282c67587e43d+413 0:413:submit_wf.cwl\n',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'name': 'submit_wf.cwl',
}, ensure_unique_name=True),
@@ -278,6 +287,11 @@ class TestCreateTemplate(unittest.TestCase):
'type': 'Directory',
'value': '99999999999999999999999999999998+99',
}
+ expect_component['script_parameters']['z'] = {
+ 'dataclass': 'Collection',
+ 'required': True,
+ 'type': 'Directory',
+ }
expect_template = {
"components": {
"submit_wf.cwl": expect_component,
diff --git a/sdk/cwl/tests/wf/expect_packed.cwl b/sdk/cwl/tests/wf/expect_packed.cwl
index f210537..25d02b2 100644
--- a/sdk/cwl/tests/wf/expect_packed.cwl
+++ b/sdk/cwl/tests/wf/expect_packed.cwl
@@ -19,6 +19,13 @@ $graph:
- default: {basename: 99999999999999999999999999999998+99, class: Directory, location: 'keep:99999999999999999999999999999998+99'}
id: '#main/y'
type: Directory
+ - default:
+ basename: anonymous
+ class: Directory
+ listing:
+ - {basename: renamed.txt, class: File, location: 'keep:99999999999999999999999999999998+99/file1.txt'}
+ id: '#main/z'
+ type: Directory
outputs: []
steps:
- id: '#main/step1'
diff --git a/sdk/cwl/tests/wf/submit_wf.cwl b/sdk/cwl/tests/wf/submit_wf.cwl
index b444655..9aab5cf 100644
--- a/sdk/cwl/tests/wf/submit_wf.cwl
+++ b/sdk/cwl/tests/wf/submit_wf.cwl
@@ -10,6 +10,8 @@ inputs:
type: File
- id: y
type: Directory
+ - id: z
+ type: Directory
outputs: []
steps:
- id: step1
commit 04f0197d84cbde94a487ab695357befa2f47a7de
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Sep 27 13:42:04 2016 -0400
Fix #10133, use "resolved" (keep reference) instead of target (filesystem
path). I think this was a merge conflict mistake.
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 8392392..4e91d66 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -85,7 +85,7 @@ def upload_dependencies(arvrunner, name, document_loader,
def setloc(p):
if not p["location"].startswith("_:") and not p["location"].startswith("keep:"):
- p["location"] = mapper.mapper(p["location"]).target
+ p["location"] = mapper.mapper(p["location"]).resolved
adjustFileObjs(workflowobj, setloc)
adjustDirObjs(workflowobj, setloc)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list