[ARVADOS] updated: 0d62edcb9d25bf4dcdb20d8872ea7b438e12fc59
Git user
git at public.curoverse.com
Fri Dec 9 14:20:33 EST 2016
Summary of changes:
.../app/controllers/work_units_controller.rb | 14 +-
apps/workbench/app/models/proxy_work_unit.rb | 6 +-
.../pipeline_instances/_running_component.html.erb | 2 +-
.../_show_components_running.html.erb | 2 +-
apps/workbench/test/integration/work_units_test.rb | 8 +-
sdk/cli/bin/crunch-job | 6 +-
sdk/cwl/arvados_cwl/__init__.py | 18 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 5 +-
sdk/cwl/arvados_cwl/arvjob.py | 8 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 9 +-
sdk/cwl/arvados_cwl/runner.py | 8 +-
sdk/cwl/tests/test_submit.py | 211 +++++++++++++++++++--
services/api/test/fixtures/pipeline_templates.yml | 22 +++
services/api/test/fixtures/workflows.yml | 19 ++
services/keepstore/s3_volume_test.go | 117 +++++++++++-
.../arvnodeman/computenode/driver/gce.py | 3 +-
16 files changed, 401 insertions(+), 57 deletions(-)
via 0d62edcb9d25bf4dcdb20d8872ea7b438e12fc59 (commit)
via 13f7a114f9c03b1efa0d6f696a7f6034b6cc2677 (commit)
via e336e28f4ce9c13aad98d059d2befb505bff365b (commit)
via 6e4f0a8e8cfe7154fe88653c48e1d4980efbdeeb (commit)
via 08271aa69aafdc92e2e8f8f25ed7fab078deb993 (commit)
via b8e3dea8979a7dbbdb62b8ecd47f7094a7f0863c (commit)
via 177873e861d0f6975832a17cfbb9d1e21acc5cce (commit)
via 9b0654adfffaac018395de29f6e441b843d46e85 (commit)
via ba94fb47b5553b85dad8544a3d8d02dcadc90c91 (commit)
via 088314afe2987920628b9e921e7de18a2bdd5745 (commit)
via 9c1a28719df89a68b83cee07e3e0ab87c1712f69 (commit)
via 3e6ad647c3da4cb77299249d1fa25767dc66b281 (commit)
via b0f9d8dcc4cee11e4faed5a6bba593819209d7e2 (commit)
via 3d30e1a5c2b4edac70628a551ed0a34f7cf3be68 (commit)
via 4876169ec54cdfe24edd3b17475b6984274b086f (commit)
via bdbcc4024e4320e17634fdfbdeb48a6051b0fccd (commit)
via cf6a9fc619a6c65d2bccb563c61b8a448334213c (commit)
via 9e06ce4eff850ee428f235501cede643f094cf6d (commit)
via e675118bd2b28ec40833d06ea384b6f1c78f3039 (commit)
via 267898c1b23b78d433ef01ddd2da8a444616e088 (commit)
via a8eec5a3443447ad7a31bd286ae7beabb8ca9e9d (commit)
from ae61fc4e1e7c1cbd4695bb3cc715eba5492c2b20 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 0d62edcb9d25bf4dcdb20d8872ea7b438e12fc59
Merge: 13f7a11 e336e28
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Dec 9 14:20:28 2016 -0500
Merge branch 'master' into 10576-cwl-keep-fetcher
Conflicts:
sdk/cwl/arvados_cwl/runner.py
sdk/cwl/tests/test_submit.py
diff --cc sdk/cwl/arvados_cwl/runner.py
index 1152b7a,2d13e66..dea4756
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@@ -184,13 -187,8 +187,14 @@@ class Runner(object)
pass
def arvados_job_spec(self, *args, **kwargs):
- self.name = self.tool.tool.get("label") or os.path.basename(self.tool.tool["id"])
+ if self.name is None:
- self.name = os.path.basename(self.tool.tool["id"])
++ self.name = self.tool.tool.get("label") or os.path.basename(self.tool.tool["id"])
+
+ # Need to filter this out, gets added by cwltool when providing
+ # parameters on the command line.
+ if "job_order" in self.job_order:
+ del self.job_order["job_order"]
+
workflowmapper = upload_instance(self.arvrunner, self.name, self.tool, self.job_order)
adjustDirObjs(self.job_order, trim_listing)
return workflowmapper
diff --cc sdk/cwl/tests/test_submit.py
index f082c71,d917aef..4418ee3
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@@ -497,165 -508,26 +516,185 @@@ class TestSubmit(unittest.TestCase)
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ @mock.patch("arvados.collection.CollectionReader")
+ @mock.patch("time.sleep")
+ @stubs
+ def test_submit_file_keepref(self, stubs, tm, collectionReader):
+ capture_stdout = cStringIO.StringIO()
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/wf/submit_keepref_wf.cwl"],
+ capture_stdout, sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 0)
+
+
+ @mock.patch("arvados.collection.CollectionReader")
+ @mock.patch("time.sleep")
+ @stubs
+ def test_submit_keepref(self, stubs, tm, reader):
+ capture_stdout = cStringIO.StringIO()
+
+ with open("tests/wf/expect_arvworkflow.cwl") as f:
+ reader().open().__enter__().read.return_value = f.read()
+
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
+ capture_stdout, sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 0)
+
+ expect_container = {
+ 'priority': 1,
+ 'mounts': {
+ '/var/spool/cwl': {
+ 'writable': True,
+ 'kind': 'collection'
+ },
+ 'stdout': {
+ 'path': '/var/spool/cwl/cwl.output.json',
+ 'kind': 'file'
+ },
+ '/var/lib/cwl/workflow': {
+ 'portable_data_hash': '99999999999999999999999999999994+99',
+ 'kind': 'collection'
+ },
+ '/var/lib/cwl/cwl.input.json': {
+ 'content': {
+ 'x': 'XxX'
+ },
+ 'kind': 'json'
+ }
+ }, 'state': 'Committed',
+ 'owner_uuid': None,
+ 'output_path': '/var/spool/cwl',
+ 'name': 'expect_arvworkflow.cwl#main',
+ 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
+ 'cwd': '/var/spool/cwl',
+ 'runtime_constraints': {
+ 'API': True,
+ 'vcpus': 1,
+ 'ram': 1073741824
+ },
+ "properties": {}
+ }
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
-
+ @mock.patch("time.sleep")
+ @stubs
+ def test_submit_arvworkflow(self, stubs, tm):
+ capture_stdout = cStringIO.StringIO()
+
+ with open("tests/wf/expect_arvworkflow.cwl") as f:
+ stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
+
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
+ capture_stdout, sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 0)
+
+ expect_container = {
+ 'priority': 1,
+ 'mounts': {
+ '/var/spool/cwl': {
+ 'writable': True,
+ 'kind': 'collection'
+ },
+ 'stdout': {
+ 'path': '/var/spool/cwl/cwl.output.json',
+ 'kind': 'file'
+ },
+ '/var/lib/cwl/workflow.json': {
+ 'kind': 'json',
+ 'json': {
+ 'cwlVersion': 'v1.0',
+ '$graph': [
+ {
+ 'inputs': [
+ {
+ 'inputBinding': {'position': 1},
+ 'type': 'string',
+ 'id': '#submit_tool.cwl/x'}
+ ],
+ 'requirements': [
+ {'dockerPull': 'debian:8', 'class': 'DockerRequirement'}
+ ],
+ 'id': '#submit_tool.cwl',
+ 'outputs': [],
+ 'baseCommand': 'cat',
+ 'class': 'CommandLineTool'
+ }, {
+ 'id': '#main',
+ 'inputs': [
+ {'type': 'string', 'id': '#main/x'}
+ ],
+ 'steps': [
+ {'in': [{'source': '#main/x', 'id': '#main/step1/x'}],
+ 'run': '#submit_tool.cwl',
+ 'id': '#main/step1',
+ 'out': []}
+ ],
+ 'class': 'Workflow',
+ 'outputs': []
+ }
+ ]
+ }
+ },
+ '/var/lib/cwl/cwl.input.json': {
+ 'content': {
+ 'x': 'XxX'
+ },
+ 'kind': 'json'
+ }
+ }, 'state': 'Committed',
+ 'owner_uuid': None,
+ 'output_path': '/var/spool/cwl',
+ 'name': 'a test workflow',
+ 'container_image': 'arvados/jobs:'+arvados_cwl.__version__,
+ 'command': ['arvados-cwl-runner', '--local', '--api=containers', '--enable-reuse', '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
+ 'cwd': '/var/spool/cwl',
+ 'runtime_constraints': {
+ 'API': True,
+ 'vcpus': 1,
+ 'ram': 1073741824
+ },
+ "properties": {
+ "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37"
+ }
+ }
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
+
+ @stubs
+ def test_submit_container_name(self, stubs):
+ capture_stdout = cStringIO.StringIO()
+ try:
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
+ "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+ capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ self.assertEqual(exited, 0)
+ except:
+ logging.exception("")
+
+ stubs.expect_container_spec["name"] = "hello container 123"
+
+ expect_container = copy.deepcopy(stubs.expect_container_spec)
+ stubs.api.container_requests().create.assert_called_with(
+ body=expect_container)
+ self.assertEqual(capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+
++
@mock.patch("arvados.commands.keepdocker.find_one_image_hash")
@mock.patch("cwltool.docker.get_image")
@mock.patch("arvados.api")
commit 13f7a114f9c03b1efa0d6f696a7f6034b6cc2677
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Dec 9 14:18:18 2016 -0500
10576: Bugfix, set self.uuid on ArvadosContainer.
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 2f31602..2c2aa7f 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -118,7 +118,8 @@ class ArvadosContainer(object):
body=container_request
).execute(num_retries=self.arvrunner.num_retries)
- self.arvrunner.processes[response["uuid"]] = self
+ self.uuid = response["uuid"]
+ self.arvrunner.processes[self.uuid] = self
logger.info("Container request %s (%s) state is %s", self.name, response["uuid"], response["state"])
@@ -254,7 +255,7 @@ class RunnerContainer(Runner):
).execute(num_retries=self.arvrunner.num_retries)
self.uuid = response["uuid"]
- self.arvrunner.processes[response["uuid"]] = self
+ self.arvrunner.processes[self.uuid] = self
logger.info("Submitted container %s", response["uuid"])
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list