[ARVADOS] created: ecf3cf1ec8544cffacc620919e35404e086b931a
Git user
git at public.curoverse.com
Mon Sep 11 08:11:44 EDT 2017
at ecf3cf1ec8544cffacc620919e35404e086b931a (commit)
commit ecf3cf1ec8544cffacc620919e35404e086b931a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Aug 29 12:56:55 2017 -0400
12186: Filter nameroot from runner-submitted input object.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 769a63b..78086ed 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -14,14 +14,14 @@ import ruamel.yaml as yaml
from cwltool.errors import WorkflowException
from cwltool.process import get_feature, UnsupportedRequirement, shortname
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
from cwltool.utils import aslist
import arvados.collection
from .arvdocker import arv_docker_get_image
from . import done
-from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location
+from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location, remove_redundant_fields
from .fsaccess import CollectionFetcher
from .pathmapper import NoFollowPathMapper, trim_listing
from .perf import Perf
@@ -286,8 +286,8 @@ class RunnerContainer(Runner):
"""
adjustDirObjs(self.job_order, trim_listing)
- adjustFileObjs(self.job_order, trim_anonymous_location)
- adjustDirObjs(self.job_order, trim_anonymous_location)
+ visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location)
+ visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields)
container_req = {
"owner_uuid": self.arvrunner.project_uuid,
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 64cd2aa..d104d56 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -13,7 +13,7 @@ from cwltool.errors import WorkflowException
from cwltool.draft2tool import revmap_file, CommandLineTool
from cwltool.load_tool import fetch_document
from cwltool.builder import Builder
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
from schema_salad.sourceline import SourceLine
@@ -23,7 +23,7 @@ import arvados.collection
from arvados.errors import ApiError
from .arvdocker import arv_docker_get_image
-from .runner import Runner, arvados_jobs_image, packed_workflow, upload_workflow_collection, trim_anonymous_location
+from .runner import Runner, arvados_jobs_image, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields
from .pathmapper import VwdPathMapper, trim_listing
from .perf import Perf
from . import done
@@ -282,8 +282,8 @@ class RunnerJob(Runner):
self.job_order["cwl:tool"] = "%s/workflow.cwl#main" % wf_pdh
adjustDirObjs(self.job_order, trim_listing)
- adjustFileObjs(self.job_order, trim_anonymous_location)
- adjustDirObjs(self.job_order, trim_anonymous_location)
+ visit_class(self.job_order, ("File", "Directory"), trim_anonymous_location)
+ visit_class(self.job_order, ("File", "Directory"), remove_redundant_fields)
if self.output_name:
self.job_order["arv:output_name"] = self.output_name
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 687bead..20cb467 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -17,7 +17,7 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
import ruamel.yaml as yaml
-from .runner import upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location
+from .runner import upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields
from .pathmapper import trim_listing
from .arvtool import ArvadosCommandTool
from .perf import Perf
@@ -113,9 +113,7 @@ class ArvadosWorkflow(Workflow):
joborder_keepmount = copy.deepcopy(joborder)
def keepmount(obj):
- for field in ("path", "nameext", "nameroot", "dirname"):
- if field in obj:
- del obj[field]
+ remove_redundant_fields(obj)
with SourceLine(obj, None, WorkflowException):
if "location" not in obj:
raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index bb4fac2..3949709 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -46,6 +46,11 @@ def trim_anonymous_location(obj):
if obj.get("location", "").startswith("_:"):
del obj["location"]
+def remove_redundant_fields(obj):
+ for field in ("path", "nameext", "nameroot", "dirname"):
+ if field in obj:
+ del obj[field]
+
def find_defaults(d, op):
if isinstance(d, list):
for i in d:
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 49545a8..0314720 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -157,8 +157,6 @@ def stubs(func):
'basename': 'blorp.txt',
'class': 'File',
'location': 'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
- "nameext": ".txt",
- "nameroot": "blorp",
"size": 16
}},
'z': {"value": {'basename': 'anonymous', 'class': 'Directory',
@@ -166,9 +164,7 @@ def stubs(func):
{
'basename': 'renamed.txt',
'class': 'File', 'location':
- 'keep:99999999999999999999999999999998+99/file1.txt',
- "nameext": ".txt",
- "nameroot": "renamed"
+ 'keep:99999999999999999999999999999998+99/file1.txt'
}
]}},
'cwl:tool': '3fffdeaa75e018172e1b583425f4ebff+60/workflow.cwl#main',
@@ -223,16 +219,12 @@ def stubs(func):
'basename': u'blorp.txt',
'class': 'File',
'location': u'keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt',
- "nameext": ".txt",
- "nameroot": "blorp",
"size": 16
},
'z': {'basename': 'anonymous', 'class': 'Directory', 'listing': [
{'basename': 'renamed.txt',
'class': 'File',
- 'location': 'keep:99999999999999999999999999999998+99/file1.txt',
- "nameext": ".txt",
- "nameroot": "renamed"
+ 'location': 'keep:99999999999999999999999999999998+99/file1.txt'
}
]}
},
commit 45054326217ef62eeef9bcdbd0ed429c55650609
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Aug 29 12:48:59 2017 -0400
12186: Filter nameroot/nameext from RunInSingleContainer and update test.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 1361579..687bead 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -13,7 +13,7 @@ from cwltool.pack import pack
from cwltool.load_tool import fetch_document
from cwltool.process import shortname
from cwltool.workflow import Workflow, WorkflowException
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
import ruamel.yaml as yaml
@@ -113,6 +113,9 @@ class ArvadosWorkflow(Workflow):
joborder_keepmount = copy.deepcopy(joborder)
def keepmount(obj):
+ for field in ("path", "nameext", "nameroot", "dirname"):
+ if field in obj:
+ del obj[field]
with SourceLine(obj, None, WorkflowException):
if "location" not in obj:
raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
@@ -126,8 +129,7 @@ class ArvadosWorkflow(Workflow):
else:
raise WorkflowException("Location is not a keep reference or a literal: '%s'" % obj["location"])
- adjustFileObjs(joborder_keepmount, keepmount)
- adjustDirObjs(joborder_keepmount, keepmount)
+ visit_class(joborder_keepmount, ("File", "Directory"), keepmount)
if self.wf_pdh is None:
adjustFileObjs(packed, keepmount)
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 5d140ba..faac1f7 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -306,9 +306,10 @@ class TestJob(unittest.TestCase):
class TestWorkflow(unittest.TestCase):
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ @mock.patch("arvados.collection.CollectionReader")
@mock.patch("arvados.collection.Collection")
@mock.patch('arvados.commands.keepdocker.list_images_in_arv')
- def test_run(self, list_images_in_arv, mockcollection):
+ def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
arvados_cwl.add_arv_hints()
api = mock.MagicMock()
@@ -319,19 +320,25 @@ class TestWorkflow(unittest.TestCase):
list_images_in_arv.return_value = [["zzzzz-4zz18-zzzzzzzzzzzzzzz"]]
runner.api.collections().get().execute.return_vaulue = {"portable_data_hash": "99999999999999999999999999999993+99"}
+ runner.api.collections().list().execute.return_vaulue = {"items": [{"portable_data_hash": "99999999999999999999999999999993+99"}]}
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.ignore_docker_for_reuse = False
runner.num_retries = 0
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+ document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=api, fs_access=make_fs_access(""))
+ document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
+ document_loader.fetch_text = document_loader.fetcher.fetch_text
+ document_loader.check_exists = document_loader.fetcher.check_exists
+
tool, metadata = document_loader.resolve_ref("tests/wf/scatter2.cwl")
metadata["cwlVersion"] = tool["cwlVersion"]
mockcollection().portable_data_hash.return_value = "99999999999999999999999999999999+118"
- make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
- collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, work_api="jobs", avsc_names=avsc_names,
basedir="", make_fs_access=make_fs_access, loader=document_loader,
makeTool=runner.arv_make_tool, metadata=metadata)
@@ -375,6 +382,11 @@ class TestWorkflow(unittest.TestCase):
mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])
mockcollection().open().__enter__().write.assert_has_calls([mock.call(
'''{
+ "fileblub": {
+ "basename": "token.txt",
+ "class": "File",
+ "location": "/keep/99999999999999999999999999999999+118/token.txt"
+ },
"sleeptime": 5
}''')])
diff --git a/sdk/cwl/tests/wf/scatter2.cwl b/sdk/cwl/tests/wf/scatter2.cwl
index e6c002d..79c3854 100644
--- a/sdk/cwl/tests/wf/scatter2.cwl
+++ b/sdk/cwl/tests/wf/scatter2.cwl
@@ -10,6 +10,11 @@ inputs:
sleeptime:
type: int[]
default: [5]
+ fileblub:
+ type: File
+ default:
+ class: File
+ location: keep:99999999999999999999999999999999+118/token.txt
outputs:
out:
type: string[]
@@ -23,6 +28,7 @@ steps:
scatterstep:
in:
sleeptime: sleeptime
+ fileblub: fileblub
out: [out]
scatter: sleeptime
hints:
@@ -32,6 +38,7 @@ steps:
id: mysub
inputs:
sleeptime: int
+ fileblub: File
outputs:
out:
type: string
diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl
index 2a9cea5..2af1155 100644
--- a/sdk/cwl/tests/wf/scatter2_subwf.cwl
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@ -11,6 +11,10 @@
"id": "#main",
"inputs": [
{
+ "id": "#main/fileblub",
+ "type": "File"
+ },
+ {
"id": "#main/sleeptime",
"type": "int"
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list