[ARVADOS] created: 0b3b1e9d91f06966cbeecd3d4a11e1fb7e8d8434
Git user
git at public.curoverse.com
Wed Dec 28 10:26:45 EST 2016
at 0b3b1e9d91f06966cbeecd3d4a11e1fb7e8d8434 (commit)
commit 0b3b1e9d91f06966cbeecd3d4a11e1fb7e8d8434
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Dec 28 10:05:17 2016 -0500
10497: Add pathlib2 to backports
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 3839de9..08b7e21 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -109,7 +109,7 @@ case "$TARGET" in
ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
'pycurl<7.21.5' contextlib2 pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro ruamel.ordereddict
- cachecontrol requests)
+ cachecontrol requests 'pathlib2>=2.1.0')
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
;;
debian8)
@@ -120,7 +120,7 @@ case "$TARGET" in
ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro ruamel.ordereddict
- cachecontrol)
+ cachecontrol 'pathlib2>=2.1.0')
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
;;
ubuntu1204)
@@ -131,7 +131,7 @@ case "$TARGET" in
ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \
contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro isodate ruamel.ordereddict
- cachecontrol requests)
+ cachecontrol requests 'pathlib2>=2.1.0')
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
;;
ubuntu1404)
@@ -140,7 +140,7 @@ case "$TARGET" in
google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \
rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \
shellescape mistune typing avro ruamel.ordereddict
- cachecontrol)
+ cachecontrol 'pathlib2>=2.1.0')
PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client==0.37.0)
;;
centos6)
@@ -160,7 +160,7 @@ case "$TARGET" in
python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \
'rdflib>=4.2.0' shellescape mistune typing avro requests \
isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
- ruamel.ordereddict cachecontrol)
+ ruamel.ordereddict cachecontrol 'pathlib2>=2.1.0')
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
export PYCURL_SSL_LIBRARY=nss
;;
@@ -180,7 +180,7 @@ case "$TARGET" in
python-daemon==2.1.1 llfuse==0.41.1 'pbr<1.0' pyyaml \
'rdflib>=4.2.0' shellescape mistune typing avro \
isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \
- ruamel.ordereddict cachecontrol)
+ ruamel.ordereddict cachecontrol 'pathlib2>=2.1.0')
PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0)
export PYCURL_SSL_LIBRARY=nss
;;
commit 5c7c672a4cd0d8c5bd0e6c5218062214eb5751f3
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Dec 27 17:06:17 2016 -0500
10497: Pop cwl:tool parameter from job order
diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 6f84d68..9804572 100644
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -48,8 +48,6 @@ def run():
def keeppathObj(v):
v["location"] = keeppath(v["location"])
- job_order_object["cwl:tool"] = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object["cwl:tool"])
-
for k,v in job_order_object.items():
if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
job_order_object[k] = {
@@ -80,7 +78,8 @@ def run():
runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
output_name=output_name, output_tags=output_tags)
- t = load_tool(job_order_object, runner.arv_make_tool)
+ toolpath = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object.pop("cwl:tool"))
+ t = load_tool(toolpath, runner.arv_make_tool)
args = argparse.Namespace()
args.project_uuid = arvados.current_job()["owner_uuid"]
commit e675daba8125b20e45e0c121f1fdcac85fb4b360
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Dec 27 15:05:48 2016 -0500
10497: Update ruamel.yaml, schema salad, and cwltool
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index ddd0124..3839de9 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -479,7 +479,7 @@ fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFI
# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh.
# Ward, 2016-05-26
-fpm_build ruamel.yaml "" "" python 0.12.4 --python-setup-py-arguments "--single-version-externally-managed"
+fpm_build ruamel.yaml "" "" python 0.13.7 --python-setup-py-arguments "--single-version-externally-managed"
# Dependency of cwltool. Fpm doesn't produce a package with the correct version
# number unless we build it explicitly
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 8c33886..4d7caf0 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -48,8 +48,9 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20161216212910',
- 'schema-salad==2.1.20161216210732',
+ 'cwltool==1.0.20161227200419',
+ 'schema-salad==2.1.20161227191302',
+ 'ruamel.yaml==0.13.7',
'arvados-python-client>=0.1.20160826210445',
'setuptools'
],
commit 91b824a77fe19242de50e838f9f8c4fb30907fa3
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Dec 21 10:34:53 2016 -0500
10497: Use SourceLine in context manager in PathMapper. Remove unnecessary import.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 85156db..d4af0f9 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -208,7 +208,7 @@ class ArvCwlRunner(object):
raise SourceLine(obj, "stderr", UnsupportedRequirement).makeError("Stderr redirection currently not suppported with --api=containers")
for v in obj.itervalues():
self.check_features(v)
- if isinstance(obj, list):
+ elif isinstance(obj, list):
for i,v in enumerate(obj):
with SourceLine(obj, i, UnsupportedRequirement):
self.check_features(v)
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 25a8ffa..dbbd83d 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -4,8 +4,6 @@ import os
import ruamel.yaml as yaml
-from schema_salad.sourceline import SourceLine
-
from cwltool.errors import WorkflowException
from cwltool.process import get_feature, UnsupportedRequirement, shortname
from cwltool.pathmapper import adjustFiles
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index e63161d..a6b3d15 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -40,19 +40,20 @@ class ArvPathMapper(PathMapper):
# mount.
ab = abspath(src, self.input_basedir)
st = arvados.commands.run.statfile("", ab, fnPattern="keep:%s/%s")
- if isinstance(st, arvados.commands.run.UploadFile):
- uploadfiles.add((src, ab, st))
- elif isinstance(st, arvados.commands.run.ArvFile):
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File")
- elif src.startswith("_:"):
- if "contents" in srcobj:
- pass
+ with SourceLine(srcobj, "location", WorkflowException):
+ if isinstance(st, arvados.commands.run.UploadFile):
+ uploadfiles.add((src, ab, st))
+ elif isinstance(st, arvados.commands.run.ArvFile):
+ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File")
+ elif src.startswith("_:"):
+ if "contents" in srcobj:
+ pass
+ else:
+ raise WorkflowException("File literal '%s' is missing contents" % src)
+ elif src.startswith("arvwf:"):
+ self._pathmap[src] = MapperEnt(src, src, "File")
else:
- raise SourceLine(srcobj, "location", WorkflowException).makeError("File literal '%s' is missing contents" % src)
- elif src.startswith("arvwf:"):
- self._pathmap[src] = MapperEnt(src, src, "File")
- else:
- raise SourceLine(srcobj, "location", WorkflowException).makeError("Input file path '%s' is invalid" % st)
+ raise WorkflowException("Input file path '%s' is invalid" % st)
if "secondaryFiles" in srcobj:
for l in srcobj["secondaryFiles"]:
self.visit(l, uploadfiles)
commit 2ecd4749bc9b3f8be8cf41864948108068e187d8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Dec 19 15:33:40 2016 -0500
10497: Improve up front feature checking and error reporting.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index c1de2a8..85156db 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -17,6 +17,7 @@ import pkg_resources # part of setuptools
from cwltool.errors import WorkflowException
import cwltool.main
import cwltool.workflow
+import cwltool.process
import schema_salad
from schema_salad.sourceline import SourceLine
@@ -192,15 +193,25 @@ class ArvCwlRunner(object):
def add_uploaded(self, src, pair):
self.uploaded[src] = pair
- def check_writable(self, obj):
+ def check_features(self, obj):
if isinstance(obj, dict):
+ if obj.get("class") == "InitialWorkDirRequirement":
+ if self.work_api == "containers":
+ raise UnsupportedRequirement("InitialWorkDirRequirement not supported with --api=containers")
if obj.get("writable"):
raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported")
+ if obj.get("class") == "CommandLineTool":
+ if self.work_api == "containers":
+ if obj.get("stdin"):
+ raise SourceLine(obj, "stdin", UnsupportedRequirement).makeError("Stdin redirection currently not suppported with --api=containers")
+ if obj.get("stderr"):
+ raise SourceLine(obj, "stderr", UnsupportedRequirement).makeError("Stderr redirection currently not suppported with --api=containers")
for v in obj.itervalues():
- self.check_writable(v)
+ self.check_features(v)
if isinstance(obj, list):
- for v in obj:
- self.check_writable(v)
+ for i,v in enumerate(obj):
+ with SourceLine(obj, i, UnsupportedRequirement):
+ self.check_features(v)
def make_output_collection(self, name, tagsString, outputObj):
outputObj = copy.deepcopy(outputObj)
@@ -309,7 +320,7 @@ class ArvCwlRunner(object):
def arv_executor(self, tool, job_order, **kwargs):
self.debug = kwargs.get("debug")
- tool.visit(self.check_writable)
+ tool.visit(self.check_features)
self.project_uuid = kwargs.get("project_uuid")
self.pipeline = None
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 827e92d..25a8ffa 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -68,17 +68,17 @@ class ArvadosContainer(object):
}
if self.generatefiles["listing"]:
- raise SourceLine(self.tool.get_requirement("InitialWorkDirRequirement")[0], None, UnsupportedRequirement).makeError("InitialWorkDirRequirement not supported with --api=containers")
+ raise UnsupportedRequirement("InitialWorkDirRequirement not supported with --api=containers")
container_request["environment"] = {"TMPDIR": self.tmpdir, "HOME": self.outdir}
if self.environment:
container_request["environment"].update(self.environment)
if self.stdin:
- raise SourceLine(self.tool.tool, "stdin", UnsupportedRequirement).makeError("Stdin redirection currently not suppported")
+ raise UnsupportedRequirement("Stdin redirection currently not suppported")
if self.stderr:
- raise SourceLine(self.tool.tool, "stderr", UnsupportedRequirement).makeError("Stderr redirection currently not suppported")
+ raise UnsupportedRequirement("Stderr redirection currently not suppported")
if self.stdout:
mounts["stdout"] = {"kind": "file",
commit 8d4ec10fc26d93d282845c789cd61da79e4b2836
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Dec 19 13:39:46 2016 -0500
10497: Add source line reporting to errors and fix tests to work with CommentedMap/Seq behavior.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 25bddc0..c1de2a8 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -18,6 +18,7 @@ from cwltool.errors import WorkflowException
import cwltool.main
import cwltool.workflow
import schema_salad
+from schema_salad.sourceline import SourceLine
import arvados
import arvados.config
@@ -194,7 +195,7 @@ class ArvCwlRunner(object):
def check_writable(self, obj):
if isinstance(obj, dict):
if obj.get("writable"):
- raise UnsupportedRequirement("InitialWorkDir feature 'writable: true' not supported")
+ raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported")
for v in obj.itervalues():
self.check_writable(v)
if isinstance(obj, list):
@@ -629,6 +630,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None):
arvargs.conformance_test = None
arvargs.use_container = True
arvargs.relax_path_checks = True
+ arvargs.validate = None
return cwltool.main.main(args=arvargs,
stdout=stdout,
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index dbbd83d..827e92d 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -4,6 +4,8 @@ import os
import ruamel.yaml as yaml
+from schema_salad.sourceline import SourceLine
+
from cwltool.errors import WorkflowException
from cwltool.process import get_feature, UnsupportedRequirement, shortname
from cwltool.pathmapper import adjustFiles
@@ -66,17 +68,17 @@ class ArvadosContainer(object):
}
if self.generatefiles["listing"]:
- raise UnsupportedRequirement("InitialWorkDirRequirement not supported with --api=containers")
+ raise SourceLine(self.tool.get_requirement("InitialWorkDirRequirement")[0], None, UnsupportedRequirement).makeError("InitialWorkDirRequirement not supported with --api=containers")
container_request["environment"] = {"TMPDIR": self.tmpdir, "HOME": self.outdir}
if self.environment:
container_request["environment"].update(self.environment)
if self.stdin:
- raise UnsupportedRequirement("Stdin redirection currently not suppported")
+ raise SourceLine(self.tool.tool, "stdin", UnsupportedRequirement).makeError("Stdin redirection currently not suppported")
if self.stderr:
- raise UnsupportedRequirement("Stderr redirection currently not suppported")
+ raise SourceLine(self.tool.tool, "stderr", UnsupportedRequirement).makeError("Stderr redirection currently not suppported")
if self.stdout:
mounts["stdout"] = {"kind": "file",
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index 7f6ab58..88c5dd2 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -2,6 +2,8 @@ import logging
import sys
import threading
+from schema_salad.sourceline import SourceLine
+
import cwltool.docker
from cwltool.errors import WorkflowException
import arvados.commands.keepdocker
@@ -16,6 +18,8 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
+ if hasattr(dockerRequirement, 'lc'):
+ dockerRequirement.lc.data["dockerImageId"] = dockerRequirement.lc.data["dockerPull"]
global cached_lookups
global cached_lookups_lock
@@ -23,45 +27,46 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
if dockerRequirement["dockerImageId"] in cached_lookups:
return cached_lookups[dockerRequirement["dockerImageId"]]
- sp = dockerRequirement["dockerImageId"].split(":")
- image_name = sp[0]
- image_tag = sp[1] if len(sp) > 1 else None
-
- images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
- image_name=image_name,
- image_tag=image_tag)
-
- if not images:
- # Fetch Docker image if necessary.
- cwltool.docker.get_image(dockerRequirement, pull_image)
-
- # Upload image to Arvados
- args = []
- if project_uuid:
- args.append("--project-uuid="+project_uuid)
- args.append(image_name)
- if image_tag:
- args.append(image_tag)
- logger.info("Uploading Docker image %s", ":".join(args[1:]))
- try:
- arvados.commands.keepdocker.main(args, stdout=sys.stderr)
- except SystemExit as e:
- if e.code:
- raise WorkflowException("keepdocker exited with code %s" % e.code)
+ with SourceLine(dockerRequirement, "dockerImageId", WorkflowException):
+ sp = dockerRequirement["dockerImageId"].split(":")
+ image_name = sp[0]
+ image_tag = sp[1] if len(sp) > 1 else None
images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
image_name=image_name,
image_tag=image_tag)
- if not images:
- raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
-
- pdh = api_client.collections().get(uuid=images[0][0]).execute()["portable_data_hash"]
-
- with cached_lookups_lock:
- cached_lookups[dockerRequirement["dockerImageId"]] = pdh
-
- return pdh
+ if not images:
+ # Fetch Docker image if necessary.
+ cwltool.docker.get_image(dockerRequirement, pull_image)
+
+ # Upload image to Arvados
+ args = []
+ if project_uuid:
+ args.append("--project-uuid="+project_uuid)
+ args.append(image_name)
+ if image_tag:
+ args.append(image_tag)
+ logger.info("Uploading Docker image %s", ":".join(args[1:]))
+ try:
+ arvados.commands.keepdocker.main(args, stdout=sys.stderr)
+ except SystemExit as e:
+ if e.code:
+ raise WorkflowException("keepdocker exited with code %s" % e.code)
+
+ images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
+ image_name=image_name,
+ image_tag=image_tag)
+
+ if not images:
+ raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
+
+ pdh = api_client.collections().get(uuid=images[0][0]).execute()["portable_data_hash"]
+
+ with cached_lookups_lock:
+ cached_lookups[dockerRequirement["dockerImageId"]] = pdh
+
+ return pdh
def arv_docker_clear_cache():
global cached_lookups
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 04e94ae..d6055d3 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -88,7 +88,8 @@ class ArvadosJob(object):
(docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
if docker_req and kwargs.get("use_container") is not False:
if docker_req.get("dockerOutputDirectory"):
- raise UnsupportedRequirement("Option 'dockerOutputDirectory' of DockerRequirement not supported.")
+ raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
+ "Option 'dockerOutputDirectory' of DockerRequirement not supported.")
runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
else:
runtime_constraints["docker_image"] = arvados_jobs_image(self.arvrunner)
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 703bb47..8c1db3a 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -3,6 +3,8 @@ import json
import copy
import logging
+from schema_salad.sourceline import SourceLine, cmap
+
from cwltool.pack import pack
from cwltool.load_tool import fetch_document
from cwltool.process import shortname
@@ -46,7 +48,7 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
"workflow": {
"name": name,
"description": tool.tool.get("doc", ""),
- "definition":yaml.safe_dump(packed)
+ "definition":yaml.round_trip_dump(packed)
}}
if project_uuid:
body["workflow"]["owner_uuid"] = project_uuid
@@ -94,23 +96,25 @@ class ArvadosWorkflow(Workflow):
joborder_keepmount = copy.deepcopy(joborder)
def keepmount(obj):
- if "location" not in obj:
- raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
- if obj["location"].startswith("keep:"):
- obj["location"] = "/keep/" + obj["location"][5:]
- if "listing" in obj:
- del obj["listing"]
- elif obj["location"].startswith("_:"):
- del obj["location"]
- else:
- raise WorkflowException("Location is not a keep reference or a literal: '%s'" % obj["location"])
+ with SourceLine(obj, None, WorkflowException):
+ if "location" not in obj:
+ raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
+ with SourceLine(obj, "location", WorkflowException):
+ if obj["location"].startswith("keep:"):
+ obj["location"] = "/keep/" + obj["location"][5:]
+ if "listing" in obj:
+ del obj["listing"]
+ elif obj["location"].startswith("_:"):
+ del obj["location"]
+ else:
+ raise WorkflowException("Location is not a keep reference or a literal: '%s'" % obj["location"])
adjustFileObjs(joborder_keepmount, keepmount)
adjustDirObjs(joborder_keepmount, keepmount)
adjustFileObjs(packed, keepmount)
adjustDirObjs(packed, keepmount)
- wf_runner = {
+ wf_runner = cmap({
"class": "CommandLineTool",
"baseCommand": "cwltool",
"inputs": self.tool["inputs"],
@@ -121,15 +125,15 @@ class ArvadosWorkflow(Workflow):
"class": "InitialWorkDirRequirement",
"listing": [{
"entryname": "workflow.cwl",
- "entry": yaml.safe_dump(packed).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
+ "entry": yaml.round_trip_dump(packed).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
}, {
"entryname": "cwl.input.yml",
- "entry": yaml.safe_dump(joborder_keepmount).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
+ "entry": yaml.round_trip_dump(joborder_keepmount).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
}]
}],
"hints": workflowobj["hints"],
"arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"]
- }
+ })
kwargs["loader"] = self.doc_loader
kwargs["avsc_names"] = self.doc_schema
return ArvadosCommandTool(self.arvrunner, wf_runner, **kwargs).job(joborder, output_callback, **kwargs)
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 74d9481..e63161d 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -6,6 +6,8 @@ import os
import arvados.commands.run
import arvados.collection
+from schema_salad.sourceline import SourceLine
+
from cwltool.pathmapper import PathMapper, MapperEnt, abspath, adjustFileObjs, adjustDirObjs
from cwltool.workflow import WorkflowException
@@ -46,11 +48,11 @@ class ArvPathMapper(PathMapper):
if "contents" in srcobj:
pass
else:
- raise WorkflowException("File literal '%s' is missing contents" % src)
+ raise SourceLine(srcobj, "location", WorkflowException).makeError("File literal '%s' is missing contents" % src)
elif src.startswith("arvwf:"):
self._pathmap[src] = MapperEnt(src, src, "File")
else:
- raise WorkflowException("Input file path '%s' is invalid" % st)
+ raise SourceLine(srcobj, "location", WorkflowException).makeError("Input file path '%s' is invalid" % st)
if "secondaryFiles" in srcobj:
for l in srcobj["secondaryFiles"]:
self.visit(l, uploadfiles)
@@ -76,7 +78,7 @@ class ArvPathMapper(PathMapper):
with c.open(path + "/" + obj["basename"], "w") as f:
f.write(obj["contents"].encode("utf-8"))
else:
- raise WorkflowException("Don't know what to do with '%s'" % obj["location"])
+ raise SourceLine(obj, "location", WorkflowException).makeError("Don't know what to do with '%s'" % obj["location"])
def setup(self, referenced_files, basedir):
# type: (List[Any], unicode) -> None
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index dea4756..e902140 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -6,6 +6,8 @@ import json
import re
from cStringIO import StringIO
+from schema_salad.sourceline import SourceLine
+
import cwltool.draft2tool
from cwltool.draft2tool import CommandLineTool
import cwltool.workflow
@@ -112,7 +114,8 @@ def upload_docker(arvrunner, tool):
if docker_req:
if docker_req.get("dockerOutputDirectory"):
# TODO: can be supported by containers API, but not jobs API.
- raise UnsupportedRequirement("Option 'dockerOutputDirectory' of DockerRequirement not supported.")
+ raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
+ "Option 'dockerOutputDirectory' of DockerRequirement not supported.")
arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid)
elif isinstance(tool, cwltool.workflow.Workflow):
for s in tool.steps:
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 0394988..45e2c7c 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -7,6 +7,7 @@ import os
import functools
import cwltool.process
from schema_salad.ref_resolver import Loader
+from schema_salad.sourceline import cmap
from .matcher import JsonDiffMatcher
@@ -34,12 +35,12 @@ class TestContainer(unittest.TestCase):
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
- tool = {
+ tool = cmap({
"inputs": [],
"outputs": [],
"baseCommand": "ls",
"arguments": [{"valueFrom": "$(runtime.outdir)"}]
- }
+ })
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
basedir="", make_fs_access=make_fs_access, loader=Loader({}))
@@ -87,7 +88,7 @@ class TestContainer(unittest.TestCase):
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- tool = {
+ tool = cmap({
"inputs": [],
"outputs": [],
"hints": [{
@@ -105,7 +106,7 @@ class TestContainer(unittest.TestCase):
"partition": "blurb"
}],
"baseCommand": "ls"
- }
+ })
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
avsc_names=avsc_names, make_fs_access=make_fs_access,
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 15da596..7675e3d 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -11,6 +11,7 @@ import arvados
import arvados_cwl
import cwltool.process
from schema_salad.ref_resolver import Loader
+from schema_salad.sourceline import cmap
from .mock_discovery import get_rootDesc
from .matcher import JsonDiffMatcher
@@ -34,12 +35,12 @@ class TestJob(unittest.TestCase):
list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
- tool = {
+ tool = cmap({
"inputs": [],
"outputs": [],
"baseCommand": "ls",
"arguments": [{"valueFrom": "$(runtime.outdir)"}]
- }
+ })
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="jobs", avsc_names=avsc_names,
basedir="", make_fs_access=make_fs_access, loader=Loader({}))
@@ -305,7 +306,7 @@ class TestWorkflow(unittest.TestCase):
find_or_create=True)
mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
- mockcollection().open().__enter__().write.assert_has_calls([mock.call('{sleeptime: 5}')])
+ mockcollection().open().__enter__().write.assert_has_calls([mock.call('sleeptime: 5')])
def test_default_work_api(self):
arvados_cwl.add_arv_hints()
diff --git a/sdk/cwl/tests/wf/expect_packed.cwl b/sdk/cwl/tests/wf/expect_packed.cwl
index 1622f48..f4d60db 100644
--- a/sdk/cwl/tests/wf/expect_packed.cwl
+++ b/sdk/cwl/tests/wf/expect_packed.cwl
@@ -1,31 +1,34 @@
+cwlVersion: v1.0
$graph:
-- baseCommand: cat
- class: CommandLineTool
- id: '#submit_tool.cwl'
+- class: CommandLineTool
+ requirements:
+ - class: DockerRequirement
+ dockerPull: debian:8
inputs:
- - default: {class: File, location: 'keep:99999999999999999999999999999991+99/tool/blub.txt'}
- id: '#submit_tool.cwl/x'
- inputBinding: {position: 1}
+ - id: '#submit_tool.cwl/x'
type: File
+ default:
+ class: File
+ location: keep:99999999999999999999999999999991+99/tool/blub.txt
+ inputBinding:
+ position: 1
outputs: []
- requirements:
- - {class: DockerRequirement, dockerPull: 'debian:8'}
+ baseCommand: cat
+ id: '#submit_tool.cwl'
- class: Workflow
- id: '#main'
inputs:
- - default: {basename: blorp.txt, class: File, location: 'keep:99999999999999999999999999999991+99/input/blorp.txt'}
- id: '#main/x'
+ - id: '#main/x'
type: File
- - default: {basename: 99999999999999999999999999999998+99, class: Directory, location: 'keep:99999999999999999999999999999998+99'}
- id: '#main/y'
+ default: {class: File, location: 'keep:99999999999999999999999999999991+99/input/blorp.txt',
+ basename: blorp.txt}
+ - id: '#main/y'
type: Directory
- - default:
- basename: anonymous
- class: Directory
- listing:
- - {basename: renamed.txt, class: File, location: 'keep:99999999999999999999999999999998+99/file1.txt'}
- id: '#main/z'
+ default: {class: Directory, location: 'keep:99999999999999999999999999999998+99',
+ basename: 99999999999999999999999999999998+99}
+ - id: '#main/z'
type: Directory
+ default: {class: Directory, basename: anonymous, listing: [{basename: renamed.txt,
+ class: File, location: 'keep:99999999999999999999999999999998+99/file1.txt'}]}
outputs: []
steps:
- id: '#main/step1'
@@ -33,4 +36,4 @@ $graph:
- {id: '#main/step1/x', source: '#main/x'}
out: []
run: '#submit_tool.cwl'
-cwlVersion: v1.0
+ id: '#main'
diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl
index 0ae1cf0..daf18b1 100644
--- a/sdk/cwl/tests/wf/scatter2_subwf.cwl
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@ -1,33 +1,41 @@
+cwlVersion: v1.0
$graph:
- class: Workflow
- hints:
- - {class: 'http://arvados.org/cwl#RunInSingleContainer'}
id: '#main'
inputs:
- - {id: '#main/sleeptime', type: int}
+ - type: int
+ id: '#main/sleeptime'
outputs:
- - {id: '#main/out', outputSource: '#main/sleep1/out', type: string}
- requirements:
- - {class: InlineJavascriptRequirement}
- - {class: ScatterFeatureRequirement}
- - {class: StepInputExpressionRequirement}
- - {class: SubworkflowFeatureRequirement}
+ - type: string
+ outputSource: '#main/sleep1/out'
+ id: '#main/out'
steps:
- - id: '#main/sleep1'
- in:
- - {id: '#main/sleep1/blurb', valueFrom: "${\n return String(inputs.sleeptime)\
- \ + \"b\";\n}\n"}
- - {id: '#main/sleep1/sleeptime', source: '#main/sleeptime'}
+ - in:
+ - valueFrom: |
+ ${
+ return String(inputs.sleeptime) + "b";
+ }
+ id: '#main/sleep1/blurb'
+ - source: '#main/sleeptime'
+ id: '#main/sleep1/sleeptime'
out: ['#main/sleep1/out']
run:
- baseCommand: sleep
class: CommandLineTool
inputs:
- - id: '#main/sleep1/sleeptime'
+ - type: int
inputBinding: {position: 1}
- type: int
+ id: '#main/sleep1/sleeptime'
outputs:
- - id: '#main/sleep1/out'
- outputBinding: {outputEval: out}
- type: string
-cwlVersion: v1.0
\ No newline at end of file
+ - type: string
+ outputBinding:
+ outputEval: out
+ id: '#main/sleep1/out'
+ baseCommand: sleep
+ id: '#main/sleep1'
+ requirements:
+ - {class: InlineJavascriptRequirement}
+ - {class: ScatterFeatureRequirement}
+ - {class: StepInputExpressionRequirement}
+ - {class: SubworkflowFeatureRequirement}
+ hints:
+ - class: http://arvados.org/cwl#RunInSingleContainer
\ No newline at end of file
commit 692127b79071f419ef39fb4594a4193f8e4a7a37
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Dec 16 16:34:59 2016 -0500
10497: Update cwltool & schema salad versions.
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index a8f732f..8c33886 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -48,8 +48,8 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20161207161158',
- 'schema-salad==1.21.20161215163938',
+ 'cwltool==1.0.20161216212910',
+ 'schema-salad==2.1.20161216210732',
'arvados-python-client>=0.1.20160826210445',
'setuptools'
],
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list