[ARVADOS] created: c627686e99abcc8508323dc6c1a6fae8507f9b2f
Git user
git at public.curoverse.com
Mon Mar 27 16:30:00 EDT 2017
at c627686e99abcc8508323dc6c1a6fae8507f9b2f (commit)
commit c627686e99abcc8508323dc6c1a6fae8507f9b2f
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Mar 27 16:27:32 2017 -0400
10401: Fixup file/directory literal handling
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index e883208..be114eb 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -52,12 +52,12 @@ class ArvPathMapper(PathMapper):
else:
raise WorkflowException("Input file path '%s' is invalid" % st)
elif src.startswith("_:"):
- if "contents" in srcobj:
- pass
- else:
- raise WorkflowException("File literal '%s' is missing contents" % src)
+ if srcobj["class"] == "File" and "contents" not in srcobj:
+ raise WorkflowException("File literal '%s' is missing `contents`" % src)
+ if srcobj["class"] == "Directory" and "listing" not in srcobj:
+ raise WorkflowException("Directory literal '%s' is missing `listing`" % src)
else:
- self._pathmap[src] = MapperEnt(src, src, "File", True)
+ self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
with SourceLine(srcobj, "secondaryFiles", WorkflowException):
for l in srcobj.get("secondaryFiles", []):
commit 0582da72a3760627cbe2433aaee27625581480f3
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Mar 27 16:18:34 2017 -0400
10401: Only upload file:// identifiers.
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 3a0c69b..e883208 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -37,26 +37,27 @@ class ArvPathMapper(PathMapper):
self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], srcobj["class"], True)
if src not in self._pathmap:
- # Local FS ref, may need to be uploaded or may be on keep
- # mount.
- ab = abspath(src, self.input_basedir)
- st = arvados.commands.run.statfile("", ab,
- fnPattern="keep:%s/%s",
- dirPattern="keep:%s/%s")
- with SourceLine(srcobj, "location", WorkflowException):
- if isinstance(st, arvados.commands.run.UploadFile):
- uploadfiles.add((src, ab, st))
- elif isinstance(st, arvados.commands.run.ArvFile):
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File", True)
- elif src.startswith("_:"):
- if "contents" in srcobj:
- pass
+ if src.startswith("file:"):
+ # Local FS ref, may need to be uploaded or may be on keep
+ # mount.
+ ab = abspath(src, self.input_basedir)
+ st = arvados.commands.run.statfile("", ab,
+ fnPattern="keep:%s/%s",
+ dirPattern="keep:%s/%s")
+ with SourceLine(srcobj, "location", WorkflowException):
+ if isinstance(st, arvados.commands.run.UploadFile):
+ uploadfiles.add((src, ab, st))
+ elif isinstance(st, arvados.commands.run.ArvFile):
+ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File", True)
else:
- raise WorkflowException("File literal '%s' is missing contents" % src)
- elif src.startswith("arvwf:"):
- self._pathmap[src] = MapperEnt(src, src, "File", True)
+ raise WorkflowException("Input file path '%s' is invalid" % st)
+ elif src.startswith("_:"):
+ if "contents" in srcobj:
+ pass
else:
- raise WorkflowException("Input file path '%s' is invalid" % st)
+ raise WorkflowException("File literal '%s' is missing contents" % src)
+ else:
+ self._pathmap[src] = MapperEnt(src, src, "File", True)
with SourceLine(srcobj, "secondaryFiles", WorkflowException):
for l in srcobj.get("secondaryFiles", []):
commit eafc2f38fcbe4906babe0616448d7d8c7d159a84
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Mar 27 16:04:55 2017 -0400
10401: Bump arvados-python-client version for updates to
arvados.command.run.uploadfiles.
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index b6d8244..ac331fa 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -51,7 +51,7 @@ setup(name='arvados-cwl-runner',
'cwltool==1.0.20170327143622',
'schema-salad==2.5.20170327140858',
'ruamel.yaml==0.13.7',
- 'arvados-python-client>=0.1.20170324133136',
+ 'arvados-python-client>=0.1.20170327195441',
'setuptools'
],
data_files=[
commit 7563a5bd4db97033230910fb5ee3bc2e63cb1393
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Mar 24 16:32:39 2017 -0400
10401: Use custom_schema_callback. Bump schema-salad and cwltool versions.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index d587f77..ed3a995 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -636,7 +636,8 @@ def add_arv_hints():
"http://arvados.org/cwl#OutputDirType",
"http://arvados.org/cwl#RuntimeConstraints",
"http://arvados.org/cwl#PartitionRequirement",
- "http://arvados.org/cwl#APIRequirement"
+ "http://arvados.org/cwl#APIRequirement",
+ "http://commonwl.org/cwltool#LoadListingRequirement"
])
@@ -720,4 +721,5 @@ def main(args, stdout, stderr, api_client=None, keep_client=None):
keep_client=keep_client,
num_retries=runner.num_retries),
resolver=partial(collectionResolver, api_client, num_retries=runner.num_retries),
- logger_handler=arvados.log_handler)
+ logger_handler=arvados.log_handler,
+ custom_schema_callback=add_arv_hints)
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 46e0898..b6d8244 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -48,8 +48,8 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20170213175853',
- 'schema-salad==2.2.20170208112505',
+ 'cwltool==1.0.20170327143622',
+ 'schema-salad==2.5.20170327140858',
'ruamel.yaml==0.13.7',
'arvados-python-client>=0.1.20170324133136',
'setuptools'
commit a3d6e83a69e1929b36b60b22c9efc5e46d0e2d35
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Mar 24 14:55:02 2017 -0400
10401: Rework support for uploading Directories (don't assume files are
enumerated).
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index e753a18..3a0c69b 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -30,36 +30,38 @@ class ArvPathMapper(PathMapper):
def visit(self, srcobj, uploadfiles):
src = srcobj["location"]
- if srcobj["class"] == "File":
- if "#" in src:
- src = src[:src.index("#")]
- if isinstance(src, basestring) and ArvPathMapper.pdh_path.match(src):
- self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "File", True)
- if src not in self._pathmap:
- # Local FS ref, may need to be uploaded or may be on keep
- # mount.
- ab = abspath(src, self.input_basedir)
- st = arvados.commands.run.statfile("", ab, fnPattern="keep:%s/%s")
- with SourceLine(srcobj, "location", WorkflowException):
- if isinstance(st, arvados.commands.run.UploadFile):
- uploadfiles.add((src, ab, st))
- elif isinstance(st, arvados.commands.run.ArvFile):
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File", True)
- elif src.startswith("_:"):
- if "contents" in srcobj:
- pass
- else:
- raise WorkflowException("File literal '%s' is missing contents" % src)
- elif src.startswith("arvwf:"):
- self._pathmap[src] = MapperEnt(src, src, "File", True)
+ if "#" in src:
+ src = src[:src.index("#")]
+
+ if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
+ self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], srcobj["class"], True)
+
+ if src not in self._pathmap:
+ # Local FS ref, may need to be uploaded or may be on keep
+ # mount.
+ ab = abspath(src, self.input_basedir)
+ st = arvados.commands.run.statfile("", ab,
+ fnPattern="keep:%s/%s",
+ dirPattern="keep:%s/%s")
+ with SourceLine(srcobj, "location", WorkflowException):
+ if isinstance(st, arvados.commands.run.UploadFile):
+ uploadfiles.add((src, ab, st))
+ elif isinstance(st, arvados.commands.run.ArvFile):
+ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File", True)
+ elif src.startswith("_:"):
+ if "contents" in srcobj:
+ pass
else:
- raise WorkflowException("Input file path '%s' is invalid" % st)
- if "secondaryFiles" in srcobj:
- for l in srcobj["secondaryFiles"]:
- self.visit(l, uploadfiles)
- elif srcobj["class"] == "Directory":
- if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
- self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "Directory", True)
+ raise WorkflowException("File literal '%s' is missing contents" % src)
+ elif src.startswith("arvwf:"):
+ self._pathmap[src] = MapperEnt(src, src, "File", True)
+ else:
+ raise WorkflowException("Input file path '%s' is invalid" % st)
+
+ with SourceLine(srcobj, "secondaryFiles", WorkflowException):
+ for l in srcobj.get("secondaryFiles", []):
+ self.visit(l, uploadfiles)
+ with SourceLine(srcobj, "listing", WorkflowException):
for l in srcobj.get("listing", []):
self.visit(l, uploadfiles)
@@ -72,7 +74,7 @@ class ArvPathMapper(PathMapper):
for l in obj.get("secondaryFiles", []):
self.addentry(l, c, path, subdirs)
elif obj["class"] == "Directory":
- for l in obj["listing"]:
+ for l in obj.get("listing", []):
self.addentry(l, c, path + "/" + obj["basename"], subdirs)
subdirs.append((obj["location"], path + "/" + obj["basename"]))
elif obj["location"].startswith("_:") and "contents" in obj:
@@ -105,17 +107,18 @@ class ArvPathMapper(PathMapper):
project=self.arvrunner.project_uuid)
for src, ab, st in uploadfiles:
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File", True)
+ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.keepref,
+ "Directory" if os.path.isdir(ab) else "File", True)
self.arvrunner.add_uploaded(src, self._pathmap[src])
for srcobj in referenced_files:
+ subdirs = []
if srcobj["class"] == "Directory":
if srcobj["location"] not in self._pathmap:
c = arvados.collection.Collection(api_client=self.arvrunner.api,
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
- subdirs = []
- for l in srcobj["listing"]:
+ for l in srcobj.get("listing", []):
self.addentry(l, c, ".", subdirs)
check = self.arvrunner.api.collections().list(filters=[["portable_data_hash", "=", c.portable_data_hash()]], limit=1).execute(num_retries=self.arvrunner.num_retries)
@@ -123,17 +126,13 @@ class ArvPathMapper(PathMapper):
c.save_new(owner_uuid=self.arvrunner.project_uuid)
ab = self.collection_pattern % c.portable_data_hash()
- self._pathmap[srcobj["location"]] = MapperEnt(ab, ab, "Directory", True)
- for loc, sub in subdirs:
- ab = self.file_pattern % (c.portable_data_hash(), sub[2:])
- self._pathmap[loc] = MapperEnt(ab, ab, "Directory", True)
+ self._pathmap[srcobj["location"]] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
elif srcobj["class"] == "File" and (srcobj.get("secondaryFiles") or
(srcobj["location"].startswith("_:") and "contents" in srcobj)):
c = arvados.collection.Collection(api_client=self.arvrunner.api,
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries )
- subdirs = []
self.addentry(srcobj, c, ".", subdirs)
check = self.arvrunner.api.collections().list(filters=[["portable_data_hash", "=", c.portable_data_hash()]], limit=1).execute(num_retries=self.arvrunner.num_retries)
@@ -141,13 +140,18 @@ class ArvPathMapper(PathMapper):
c.save_new(owner_uuid=self.arvrunner.project_uuid)
ab = self.file_pattern % (c.portable_data_hash(), srcobj["basename"])
- self._pathmap[srcobj["location"]] = MapperEnt(ab, ab, "File", True)
+ self._pathmap[srcobj["location"]] = MapperEnt("keep:%s/%s" % (c.portable_data_hash(), srcobj["basename"]),
+ ab, "File", True)
if srcobj.get("secondaryFiles"):
ab = self.collection_pattern % c.portable_data_hash()
- self._pathmap["_:" + unicode(uuid.uuid4())] = MapperEnt(ab, ab, "Directory", True)
+ self._pathmap["_:" + unicode(uuid.uuid4())] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
+
+ if subdirs:
for loc, sub in subdirs:
+ # subdirs will all start with "./", strip it off
ab = self.file_pattern % (c.portable_data_hash(), sub[2:])
- self._pathmap[loc] = MapperEnt(ab, ab, "Directory", True)
+ self._pathmap[loc] = MapperEnt("keep:%s/%s" % (c.portable_data_hash(), sub[2:]),
+ ab, "Directory", True)
self.keepdir = None
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index d5a75a9..46e0898 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -51,7 +51,7 @@ setup(name='arvados-cwl-runner',
'cwltool==1.0.20170213175853',
'schema-salad==2.2.20170208112505',
'ruamel.yaml==0.13.7',
- 'arvados-python-client>=0.1.20170112173420',
+ 'arvados-python-client>=0.1.20170324133136',
'setuptools'
],
data_files=[
diff --git a/sdk/cwl/tests/test_pathmapper.py b/sdk/cwl/tests/test_pathmapper.py
index 6b5e4a8..a81a853 100644
--- a/sdk/cwl/tests/test_pathmapper.py
+++ b/sdk/cwl/tests/test_pathmapper.py
@@ -19,6 +19,7 @@ from arvados_cwl.pathmapper import ArvPathMapper
def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPattern="$(file %s/%s)", name=None):
pdh = "99999999999999999999999999999991+99"
for c in files:
+ c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn))
c.fn = fnPattern % (pdh, os.path.basename(c.fn))
class TestPathmap(unittest.TestCase):
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index cab18dc..85c49c9 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -249,17 +249,17 @@ class TestSubmit(unittest.TestCase):
mock.call(body=JsonDiffMatcher({
'manifest_text':
'. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
- 'owner_uuid': None,
+ 'replication_desired': None,
'name': 'submit_tool.cwl dependencies',
}), ensure_unique_name=True),
- mock.call().execute(),
+ mock.call().execute(num_retries=4),
mock.call(body=JsonDiffMatcher({
'manifest_text':
'. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
- 'owner_uuid': None,
+ 'replication_desired': None,
'name': 'submit_wf.cwl input',
}), ensure_unique_name=True),
- mock.call().execute()])
+ mock.call().execute(num_retries=4)])
arvdock.assert_has_calls([
mock.call(stubs.api, {"class": "DockerRequirement", "dockerPull": "debian:8"}, True, None),
@@ -432,17 +432,17 @@ class TestSubmit(unittest.TestCase):
mock.call(body=JsonDiffMatcher({
'manifest_text':
'. 5bcc9fe8f8d5992e6cf418dc7ce4dbb3+16 0:16:blub.txt\n',
- 'owner_uuid': None,
+ 'replication_desired': None,
'name': 'submit_tool.cwl dependencies',
}), ensure_unique_name=True),
- mock.call().execute(),
+ mock.call().execute(num_retries=4),
mock.call(body=JsonDiffMatcher({
'manifest_text':
'. 979af1245a12a1fed634d4222473bfdc+16 0:16:blorp.txt\n',
- 'owner_uuid': None,
+ 'replication_desired': None,
'name': 'submit_wf.cwl input',
}), ensure_unique_name=True),
- mock.call().execute()])
+ mock.call().execute(num_retries=4)])
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
@@ -822,18 +822,8 @@ class TestSubmit(unittest.TestCase):
arvrunner.project_uuid = ""
api.return_value = mock.MagicMock()
arvrunner.api = api.return_value
- arvrunner.api.links().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
- {"items": [], "items_available": 0, "offset": 0},
- {"items": [], "items_available": 0, "offset": 0},
- {"items": [{"created_at": "",
- "head_uuid": "",
- "link_class": "docker_image_hash",
- "name": "123456",
- "owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0},
- {"items": [], "items_available": 0, "offset": 0},
- {"items": [{"created_at": "",
- "head_uuid": "",
+ arvrunner.api.links().list().execute.side_effect = ({"items": [{"created_at": "",
+ "head_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
"link_class": "docker_image_repo+tag",
"name": "arvados/jobs:"+arvados_cwl.__version__,
"owner_uuid": "",
@@ -843,19 +833,18 @@ class TestSubmit(unittest.TestCase):
"link_class": "docker_image_hash",
"name": "123456",
"owner_uuid": "",
- "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0} ,
+ "properties": {"image_timestamp": ""}}], "items_available": 1, "offset": 0}
)
find_one_image_hash.return_value = "123456"
- arvrunner.api.collections().list().execute.side_effect = ({"items": [], "items_available": 0, "offset": 0},
- {"items": [{"uuid": "",
+ arvrunner.api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzb",
"owner_uuid": "",
"manifest_text": "",
"properties": ""
- }], "items_available": 1, "offset": 0},
- {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
+ }], "items_available": 1, "offset": 0},)
arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
- self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
+ self.assertEqual("arvados/jobs:"+arvados_cwl.__version__,
+ arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
class TestCreateTemplate(unittest.TestCase):
existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"
commit 869a759f43ca2f8c6171106ecbfb897d71eea07a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Mar 8 13:23:38 2017 -0500
10401: Allow Arvados extensions to be used in "requirements" not just "hints".
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 528d1b6..d587f77 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -37,8 +37,8 @@ from .pathmapper import NoFollowPathMapper
from ._version import __version__
from cwltool.pack import pack
-from cwltool.process import shortname, UnsupportedRequirement, getListing, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
from cwltool.draft2tool import compute_checksums
from arvados.api import OrderedJsonModel
@@ -520,7 +520,7 @@ class ArvCwlRunner(object):
self.set_crunch_output()
if kwargs.get("compute_checksum"):
- adjustDirObjs(self.final_output, partial(getListing, self.fs_access))
+ adjustDirObjs(self.final_output, partial(get_listing, self.fs_access))
adjustFileObjs(self.final_output, partial(compute_checksums, self.fs_access))
return (self.final_output, self.final_status)
@@ -631,6 +631,14 @@ def add_arv_hints():
res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema.yml')
use_custom_schema("v1.0", "http://arvados.org/cwl", res.read())
res.close()
+ cwltool.process.supportedProcessRequirements.extend([
+ "http://arvados.org/cwl#RunInSingleContainer",
+ "http://arvados.org/cwl#OutputDirType",
+ "http://arvados.org/cwl#RuntimeConstraints",
+ "http://arvados.org/cwl#PartitionRequirement",
+ "http://arvados.org/cwl#APIRequirement"
+ ])
+
def main(args, stdout, stderr, api_client=None, keep_client=None):
parser = arg_parser()
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
index 4b30483..133638c 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
@@ -40,8 +40,6 @@ $graph:
- name: OutputDirType
type: enum
- extends: cwl:ProcessRequirement
- inVocab: false
symbols:
- local_output_dir
- keep_output_dir
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 3090936..5c697b3 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -53,7 +53,7 @@ class ArvadosContainer(object):
dirs = set()
for f in self.pathmapper.files():
- pdh, p, tp = self.pathmapper.mapper(f)
+ pdh, p, tp, stg = self.pathmapper.mapper(f)
if tp == "Directory" and '/' not in pdh:
mounts[p] = {
"kind": "collection",
@@ -62,7 +62,7 @@ class ArvadosContainer(object):
dirs.add(pdh)
for f in self.pathmapper.files():
- res, p, tp = self.pathmapper.mapper(f)
+ res, p, tp, stg = self.pathmapper.mapper(f)
if res.startswith("keep:"):
res = res[5:]
elif res.startswith("/keep/"):
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 1f6aa57..e753a18 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -34,7 +34,7 @@ class ArvPathMapper(PathMapper):
if "#" in src:
src = src[:src.index("#")]
if isinstance(src, basestring) and ArvPathMapper.pdh_path.match(src):
- self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "File")
+ self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "File", True)
if src not in self._pathmap:
# Local FS ref, may need to be uploaded or may be on keep
# mount.
@@ -44,14 +44,14 @@ class ArvPathMapper(PathMapper):
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File")
+ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File", True)
elif src.startswith("_:"):
if "contents" in srcobj:
pass
else:
raise WorkflowException("File literal '%s' is missing contents" % src)
elif src.startswith("arvwf:"):
- self._pathmap[src] = MapperEnt(src, src, "File")
+ self._pathmap[src] = MapperEnt(src, src, "File", True)
else:
raise WorkflowException("Input file path '%s' is invalid" % st)
if "secondaryFiles" in srcobj:
@@ -59,7 +59,7 @@ class ArvPathMapper(PathMapper):
self.visit(l, uploadfiles)
elif srcobj["class"] == "Directory":
if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
- self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "Directory")
+ self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "Directory", True)
for l in srcobj.get("listing", []):
self.visit(l, uploadfiles)
@@ -90,7 +90,7 @@ class ArvPathMapper(PathMapper):
loc = k["location"]
if loc in already_uploaded:
v = already_uploaded[loc]
- self._pathmap[loc] = MapperEnt(v.resolved, self.collection_pattern % v.resolved[5:], "File")
+ self._pathmap[loc] = MapperEnt(v.resolved, self.collection_pattern % v.resolved[5:], "File", True)
for srcobj in referenced_files:
self.visit(srcobj, uploadfiles)
@@ -105,7 +105,7 @@ class ArvPathMapper(PathMapper):
project=self.arvrunner.project_uuid)
for src, ab, st in uploadfiles:
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File")
+ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File", True)
self.arvrunner.add_uploaded(src, self._pathmap[src])
for srcobj in referenced_files:
@@ -123,10 +123,10 @@ class ArvPathMapper(PathMapper):
c.save_new(owner_uuid=self.arvrunner.project_uuid)
ab = self.collection_pattern % c.portable_data_hash()
- self._pathmap[srcobj["location"]] = MapperEnt(ab, ab, "Directory")
+ self._pathmap[srcobj["location"]] = MapperEnt(ab, ab, "Directory", True)
for loc, sub in subdirs:
ab = self.file_pattern % (c.portable_data_hash(), sub[2:])
- self._pathmap[loc] = MapperEnt(ab, ab, "Directory")
+ self._pathmap[loc] = MapperEnt(ab, ab, "Directory", True)
elif srcobj["class"] == "File" and (srcobj.get("secondaryFiles") or
(srcobj["location"].startswith("_:") and "contents" in srcobj)):
@@ -141,13 +141,13 @@ class ArvPathMapper(PathMapper):
c.save_new(owner_uuid=self.arvrunner.project_uuid)
ab = self.file_pattern % (c.portable_data_hash(), srcobj["basename"])
- self._pathmap[srcobj["location"]] = MapperEnt(ab, ab, "File")
+ self._pathmap[srcobj["location"]] = MapperEnt(ab, ab, "File", True)
if srcobj.get("secondaryFiles"):
ab = self.collection_pattern % c.portable_data_hash()
- self._pathmap["_:" + unicode(uuid.uuid4())] = MapperEnt(ab, ab, "Directory")
+ self._pathmap["_:" + unicode(uuid.uuid4())] = MapperEnt(ab, ab, "Directory", True)
for loc, sub in subdirs:
ab = self.file_pattern % (c.portable_data_hash(), sub[2:])
- self._pathmap[loc] = MapperEnt(ab, ab, "Directory")
+ self._pathmap[loc] = MapperEnt(ab, ab, "Directory", True)
self.keepdir = None
@@ -162,24 +162,24 @@ class ArvPathMapper(PathMapper):
class StagingPathMapper(PathMapper):
_follow_dirs = True
- def visit(self, obj, stagedir, basedir, copy=False):
+ def visit(self, obj, stagedir, basedir, copy=False, staged=False):
# type: (Dict[unicode, Any], unicode, unicode, bool) -> None
loc = obj["location"]
tgt = os.path.join(stagedir, obj["basename"])
if obj["class"] == "Directory":
- self._pathmap[loc] = MapperEnt(loc, tgt, "Directory")
+ self._pathmap[loc] = MapperEnt(loc, tgt, "Directory", staged)
if loc.startswith("_:") or self._follow_dirs:
self.visitlisting(obj.get("listing", []), tgt, basedir)
elif obj["class"] == "File":
if loc in self._pathmap:
return
if "contents" in obj and loc.startswith("_:"):
- self._pathmap[loc] = MapperEnt(obj["contents"], tgt, "CreateFile")
+ self._pathmap[loc] = MapperEnt(obj["contents"], tgt, "CreateFile", staged)
else:
if copy:
- self._pathmap[loc] = MapperEnt(loc, tgt, "WritableFile")
+ self._pathmap[loc] = MapperEnt(loc, tgt, "WritableFile", staged)
else:
- self._pathmap[loc] = MapperEnt(loc, tgt, "File")
+ self._pathmap[loc] = MapperEnt(loc, tgt, "File", staged)
self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir)
@@ -191,9 +191,9 @@ class VwdPathMapper(StagingPathMapper):
# with any secondary files.
self.visitlisting(referenced_files, self.stagedir, basedir)
- for path, (ab, tgt, type) in self._pathmap.items():
+ for path, (ab, tgt, type, staged) in self._pathmap.items():
if type in ("File", "Directory") and ab.startswith("keep:"):
- self._pathmap[path] = MapperEnt("$(task.keep)/%s" % ab[5:], tgt, type)
+ self._pathmap[path] = MapperEnt("$(task.keep)/%s" % ab[5:], tgt, type, staged)
class NoFollowPathMapper(StagingPathMapper):
diff --git a/sdk/cwl/tests/test_pathmapper.py b/sdk/cwl/tests/test_pathmapper.py
index 3b6af04..6b5e4a8 100644
--- a/sdk/cwl/tests/test_pathmapper.py
+++ b/sdk/cwl/tests/test_pathmapper.py
@@ -36,7 +36,7 @@ class TestPathmap(unittest.TestCase):
"location": "keep:99999999999999999999999999999991+99/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+ self.assertEqual({'keep:99999999999999999999999999999991+99/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
@mock.patch("arvados.commands.run.uploadfiles")
@@ -52,7 +52,7 @@ class TestPathmap(unittest.TestCase):
"location": "tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+ self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
@mock.patch("arvados.commands.run.uploadfiles")
@@ -60,7 +60,7 @@ class TestPathmap(unittest.TestCase):
"""Test pathmapper handling previously uploaded files."""
arvrunner = arvados_cwl.ArvCwlRunner(self.api)
- arvrunner.add_uploaded('tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File'))
+ arvrunner.add_uploaded('tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File', staged=True))
upl.side_effect = upload_mock
@@ -69,7 +69,7 @@ class TestPathmap(unittest.TestCase):
"location": "tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+ self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
@mock.patch("arvados.commands.run.uploadfiles")
@@ -92,5 +92,5 @@ class TestPathmap(unittest.TestCase):
"location": "tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
+ self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
commit acaa7f2747ce82b1f3df95af7a6769b305009225
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Mar 8 13:19:42 2017 -0500
10401: Use use_custom_schema feature for extensions.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 36fe6ef..528d1b6 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -37,7 +37,7 @@ from .pathmapper import NoFollowPathMapper
from ._version import __version__
from cwltool.pack import pack
-from cwltool.process import shortname, UnsupportedRequirement, getListing
+from cwltool.process import shortname, UnsupportedRequirement, getListing, use_custom_schema
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
from cwltool.draft2tool import compute_checksums
from arvados.api import OrderedJsonModel
@@ -628,16 +628,9 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
return parser
def add_arv_hints():
- cache = {}
res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema.yml')
- cache["http://arvados.org/cwl"] = res.read()
+ use_custom_schema("v1.0", "http://arvados.org/cwl", res.read())
res.close()
- document_loader, cwlnames, _, _ = cwltool.process.get_schema("v1.0")
- _, extnames, _, _ = schema_salad.schema.load_schema("http://arvados.org/cwl", cache=cache)
- for n in extnames.names:
- if not cwlnames.has_name("http://arvados.org/cwl#"+n, ""):
- cwlnames.add_name("http://arvados.org/cwl#"+n, "", extnames.get_name(n, ""))
- document_loader.idx["http://arvados.org/cwl#"+n] = {}
def main(args, stdout, stderr, api_client=None, keep_client=None):
parser = arg_parser()
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
index 3a6eb47..4b30483 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
@@ -1,7 +1,32 @@
$base: "http://arvados.org/cwl#"
+$namespaces:
+ cwl: "https://w3id.org/cwl/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
$graph:
+- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
+
+- name: cwltool:LoadListingRequirement
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ fields:
+ class:
+ type: string
+ doc: "Always 'LoadListingRequirement'"
+ jsonldPredicate:
+ "_id": "@type"
+ "_type": "@vocab"
+ loadListing:
+ type:
+ - "null"
+ - type: enum
+ name: LoadListingEnum
+ symbols: [shallow, deep]
+
- name: RunInSingleContainer
type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
doc: |
Indicates that a subworkflow should run in a single container
and not be scheduled as separate steps.
@@ -15,6 +40,8 @@ $graph:
- name: OutputDirType
type: enum
+ extends: cwl:ProcessRequirement
+ inVocab: false
symbols:
- local_output_dir
- keep_output_dir
@@ -38,6 +65,8 @@ $graph:
- name: RuntimeConstraints
type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
doc: |
Set Arvados-specific runtime hints.
fields:
@@ -62,6 +91,8 @@ $graph:
- name: PartitionRequirement
type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
doc: |
Select preferred compute partitions on which to run jobs.
fields:
@@ -72,6 +103,8 @@ $graph:
- name: APIRequirement
type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
doc: |
Indicates that process wants to access to the Arvados API. Will be granted
limited network access and have ARVADOS_API_HOST and ARVADOS_API_TOKEN set
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list