[ARVADOS] updated: 6884eedcf015af4b79857d8a47ef93f844e6d190
Git user
git at public.curoverse.com
Wed Jul 13 14:45:43 EDT 2016
Summary of changes:
sdk/cwl/arvados_cwl/__init__.py | 2 +-
sdk/cwl/arvados_cwl/arvjob.py | 8 +++----
sdk/cwl/arvados_cwl/fsaccess.py | 6 +++--
sdk/cwl/arvados_cwl/runner.py | 21 ++++++++---------
sdk/cwl/tests/test_submit.py | 46 +++++++++++++++++++++++++++-----------
sdk/cwl/tests/tool/submit_tool.cwl | 4 ++--
sdk/cwl/tests/wf/inputs_test.cwl | 10 ++++-----
sdk/cwl/tests/wf/submit_wf.cwl | 6 ++---
sdk/python/arvados/collection.py | 1 +
9 files changed, 64 insertions(+), 40 deletions(-)
via 6884eedcf015af4b79857d8a47ef93f844e6d190 (commit)
from b7b9ea44ada30b1251fb10c872cb1da1d7c29bd0 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6884eedcf015af4b79857d8a47ef93f844e6d190
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jul 13 14:45:39 2016 -0400
Fix tests. refs #9570
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 1959b5c..847da90 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -110,6 +110,7 @@ class ArvCwlRunner(object):
useruuid = self.api.users().current().execute()["uuid"]
self.project_uuid = kwargs.get("project_uuid") if kwargs.get("project_uuid") else useruuid
self.pipeline = None
+ self.fs_access = CollectionFsAccess(kwargs["basedir"], api_client=self.api)
if kwargs.get("create_template"):
tmpl = RunnerTemplate(self, tool, job_order, kwargs.get("enable_reuse"))
@@ -119,7 +120,6 @@ class ArvCwlRunner(object):
self.debug = kwargs.get("debug")
self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")
- self.fs_access = CollectionFsAccess(kwargs["basedir"])
kwargs["fs_access"] = self.fs_access
kwargs["enable_reuse"] = kwargs.get("enable_reuse")
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 6fb0fb9..15cafad 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -95,7 +95,7 @@ class ArvadosJob(object):
"owner_uuid": self.arvrunner.project_uuid,
"script": "crunchrunner",
"repository": "arvados",
- "script_version": "9570-cwl-v1.0",
+ "script_version": "master",
"minimum_script_version": "9e5b98e8f5f4727856b53447191f9c06e3da2ba6",
"script_parameters": {"tasks": [script_parameters]},
"runtime_constraints": runtime_constraints
@@ -297,7 +297,7 @@ class RunnerTemplate(object):
# Title and description...
title = param.pop('label', '')
- descr = param.pop('description', '').rstrip('\n')
+ descr = param.pop('doc', '').rstrip('\n')
if title:
param['title'] = title
if descr:
@@ -310,8 +310,8 @@ class RunnerTemplate(object):
pass
elif not isinstance(value, dict):
param['value'] = value
- elif param.get('dataclass') == 'File' and value.get('path'):
- param['value'] = value['path']
+ elif param.get('dataclass') == 'File' and value.get('location'):
+ param['value'] = value['location']
spec['script_parameters'][param_id] = param
spec['script_parameters']['cwl:tool'] = job_params['cwl:tool']
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 548b166..661a9e0 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -11,16 +11,18 @@ import arvados.arvfile
class CollectionFsAccess(cwltool.process.StdFsAccess):
"""Implement the cwltool FsAccess interface for Arvados Collections."""
- def __init__(self, basedir):
+ def __init__(self, basedir, api_client=None):
super(CollectionFsAccess, self).__init__(basedir)
+ self.api_client = api_client
self.collections = {}
def get_collection(self, path):
+ print "PPP", path
p = path.split("/")
if p[0].startswith("keep:") and arvados.util.keep_locator_pattern.match(p[0][5:]):
pdh = p[0][5:]
if pdh not in self.collections:
- self.collections[pdh] = arvados.collection.CollectionReader(pdh)
+ self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client)
return (self.collections[pdh], "/".join(p[1:]))
else:
return (None, path)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index fe451f6..19cb7ea 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -10,7 +10,7 @@ from cwltool.draft2tool import CommandLineTool
import cwltool.workflow
from cwltool.process import get_feature, scandeps, UnsupportedRequirement
from cwltool.load_tool import fetch_document
-from cwltool.pathmapper import adjustFiles
+from cwltool.pathmapper import adjustFileObjs
import arvados.collection
@@ -46,15 +46,14 @@ class Runner(object):
def arvados_job_spec(self, *args, **kwargs):
self.upload_docker(self.tool)
- workflowfiles = set()
- jobfiles = set()
- workflowfiles.add(self.tool.tool["id"])
+ workflowfiles = []
+ jobfiles = []
+ workflowfiles.append({"class":"File", "location": self.tool.tool["id"]})
self.name = os.path.basename(self.tool.tool["id"])
def visitFiles(files, path):
- files.add(path)
- return path
+ files.append(path)
document_loader, workflowobj, uri = fetch_document(self.tool.tool["id"])
loaded = set()
@@ -68,10 +67,10 @@ class Runner(object):
sc = scandeps(uri, workflowobj,
set(("$import", "run")),
- set(("$include", "$schemas", "path")),
+ set(("$include", "$schemas", "path", "location")),
loadref)
- adjustFiles(sc, partial(visitFiles, workflowfiles))
- adjustFiles(self.job_order, partial(visitFiles, jobfiles))
+ adjustFileObjs(sc, partial(visitFiles, workflowfiles))
+ adjustFileObjs(self.job_order, partial(visitFiles, jobfiles))
keepprefix = kwargs.get("keepprefix", "")
workflowmapper = ArvPathMapper(self.arvrunner, workflowfiles, "",
@@ -86,7 +85,9 @@ class Runner(object):
name=os.path.basename(self.job_order.get("id", "#")),
**kwargs)
- adjustFiles(self.job_order, lambda p: jobmapper.mapper(p)[1])
+ def setloc(p):
+ p["location"] = jobmapper.mapper(p["location"])[1]
+ adjustFileObjs(self.job_order, setloc)
if "id" in self.job_order:
del self.job_order["id"]
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 6f056ab..2dff3e8 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -41,17 +41,24 @@ def stubs(func):
stubs.api.collections().create().execute.side_effect = ({
"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
"portable_data_hash": "99999999999999999999999999999991+99",
+ "manifest_text": ""
}, {
"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
"portable_data_hash": "99999999999999999999999999999992+99",
+ "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"
},
{
"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz4",
"portable_data_hash": "99999999999999999999999999999994+99",
"manifest_text": ""
- })
+ },
+ {
+ "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz5",
+ "portable_data_hash": "99999999999999999999999999999995+99",
+ "manifest_text": ""
+ } )
stubs.api.collections().get().execute.return_value = {
- "portable_data_hash": "99999999999999999999999999999993+99"}
+ "portable_data_hash": "99999999999999999999999999999993+99", "manifest_text": "./tool 00000000000000000000000000000000+0 0:0:submit_tool.cwl 0:0:blub.txt"}
stubs.expect_job_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
stubs.api.jobs().create().execute.return_value = {
@@ -76,7 +83,8 @@ def stubs(func):
},
'script_parameters': {
'x': {
- 'path': '99999999999999999999999999999992+99/blorp.txt',
+ 'basename': 'blorp.txt',
+ 'location': '99999999999999999999999999999994+99/blorp.txt',
'class': 'File'
},
'cwl:tool':
@@ -103,7 +111,7 @@ def stubs(func):
'kind': 'file'
},
'/var/lib/cwl/job/cwl.input.json': {
- 'portable_data_hash': '33be5c865fe12e1e4788d2f1bc627f7a+60/cwl.input.json',
+ 'portable_data_hash': '765fda0d9897729ff467a4609879c00a+60/cwl.input.json',
'kind': 'collection'
}
},
@@ -138,13 +146,19 @@ class TestSubmit(unittest.TestCase):
mock.call(),
mock.call(body={
'manifest_text':
- './tool a3954c369b8924d40547ec8cf5f6a7f4+449 '
- '0:16:blub.txt 16:433:submit_tool.cwl\n./wf '
- 'e046cace0b1a0a6ee645f6ea8688f7e2+364 0:364:submit_wf.cwl\n',
+ './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
+ '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
+ '4d31c5fefd087faf67ca8db0111af36c+353 0:353:submit_wf.cwl\n',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'name': 'submit_wf.cwl',
}, ensure_unique_name=True),
mock.call().execute(),
+ mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
+ '0:0:blub.txt 0:0:submit_tool.cwl\n',
+ 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
+ 'name': 'New collection'},
+ ensure_unique_name=True),
+ mock.call().execute(num_retries=4),
mock.call(body={
'manifest_text':
'. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
@@ -191,13 +205,19 @@ class TestSubmit(unittest.TestCase):
mock.call(),
mock.call(body={
'manifest_text':
- './tool a3954c369b8924d40547ec8cf5f6a7f4+449 '
- '0:16:blub.txt 16:433:submit_tool.cwl\n./wf '
- 'e046cace0b1a0a6ee645f6ea8688f7e2+364 0:364:submit_wf.cwl\n',
+ './tool d51232d96b6116d964a69bfb7e0c73bf+450 '
+ '0:16:blub.txt 16:434:submit_tool.cwl\n./wf '
+ '4d31c5fefd087faf67ca8db0111af36c+353 0:353:submit_wf.cwl\n',
'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
'name': 'submit_wf.cwl',
}, ensure_unique_name=True),
mock.call().execute(),
+ mock.call(body={'manifest_text': '. d41d8cd98f00b204e9800998ecf8427e+0 '
+ '0:0:blub.txt 0:0:submit_tool.cwl\n',
+ 'owner_uuid': 'zzzzz-tpzed-zzzzzzzzzzzzzzz',
+ 'name': 'New collection'},
+ ensure_unique_name=True),
+ mock.call().execute(num_retries=4),
mock.call(body={
'manifest_text':
'. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
@@ -222,7 +242,7 @@ class TestCreateTemplate(unittest.TestCase):
capture_stdout = cStringIO.StringIO()
exited = arvados_cwl.main(
- ["--create-template", "--no-wait",
+ ["--create-template", "--no-wait", "--debug",
"--project-uuid", project_uuid,
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
capture_stdout, sys.stderr, api_client=stubs.api)
@@ -236,7 +256,7 @@ class TestCreateTemplate(unittest.TestCase):
'dataclass': 'File',
'required': True,
'type': 'File',
- 'value': '99999999999999999999999999999992+99/blorp.txt',
+ 'value': '99999999999999999999999999999994+99/blorp.txt',
}
expect_template = {
"components": {
@@ -327,7 +347,7 @@ class TestTemplateInputs(unittest.TestCase):
expect_template["owner_uuid"] = stubs.fake_user_uuid
params = expect_template[
"components"]["inputs_test.cwl"]["script_parameters"]
- params["fileInput"]["value"] = '99999999999999999999999999999992+99/blorp.txt'
+ params["fileInput"]["value"] = '99999999999999999999999999999994+99/blorp.txt'
params["floatInput"]["value"] = 1.234
params["boolInput"]["value"] = True
diff --git a/sdk/cwl/tests/tool/submit_tool.cwl b/sdk/cwl/tests/tool/submit_tool.cwl
index e9fa423..19df1de 100644
--- a/sdk/cwl/tests/tool/submit_tool.cwl
+++ b/sdk/cwl/tests/tool/submit_tool.cwl
@@ -4,7 +4,7 @@
# value blub.txt) and uploading to Keep works as intended.
class: CommandLineTool
-cwlVersion: draft-3
+cwlVersion: v1.0
requirements:
- class: DockerRequirement
dockerPull: debian:8
@@ -13,7 +13,7 @@ inputs:
type: File
default:
class: File
- path: blub.txt
+ location: blub.txt
inputBinding:
position: 1
outputs: []
diff --git a/sdk/cwl/tests/wf/inputs_test.cwl b/sdk/cwl/tests/wf/inputs_test.cwl
index ec43207..5fea4fd 100644
--- a/sdk/cwl/tests/wf/inputs_test.cwl
+++ b/sdk/cwl/tests/wf/inputs_test.cwl
@@ -2,12 +2,12 @@
# various input types as script_parameters in pipeline templates.
class: Workflow
-cwlVersion: draft-3
+cwlVersion: v1.0
inputs:
- id: "#fileInput"
type: File
label: It's a file; we expect to find some characters in it.
- description: |
+ doc: |
If there were anything further to say, it would be said here,
or here.
- id: "#boolInput"
@@ -22,7 +22,7 @@ inputs:
outputs: []
steps:
- id: step1
- inputs:
- - { id: x, source: "#x" }
- outputs: []
+ in:
+ - { id: x, source: "#fileInput" }
+ out: []
run: ../tool/submit_tool.cwl
diff --git a/sdk/cwl/tests/wf/submit_wf.cwl b/sdk/cwl/tests/wf/submit_wf.cwl
index 36db603..874c72c 100644
--- a/sdk/cwl/tests/wf/submit_wf.cwl
+++ b/sdk/cwl/tests/wf/submit_wf.cwl
@@ -4,14 +4,14 @@
# (e.g. submit_tool.cwl) and uploading to Keep works as intended.
class: Workflow
-cwlVersion: draft-3
+cwlVersion: v1.0
inputs:
- id: x
type: File
outputs: []
steps:
- id: step1
- inputs:
+ in:
- { id: x, source: "#x" }
- outputs: []
+ out: []
run: ../tool/submit_tool.cwl
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 8450bd1..93ee167 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1482,6 +1482,7 @@ class Collection(RichCollectionBase):
body["owner_uuid"] = owner_uuid
self._remember_api_response(self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries))
+ print self._api_response
text = self._api_response["manifest_text"]
self._manifest_locator = self._api_response["uuid"]
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list