[ARVADOS] created: f8f8b9030fd63fd6ccc10d02cd2e4cba6ea685b8
Git user
git at public.curoverse.com
Mon Jun 5 08:55:26 EDT 2017
at f8f8b9030fd63fd6ccc10d02cd2e4cba6ea685b8 (commit)
commit f8f8b9030fd63fd6ccc10d02cd2e4cba6ea685b8
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Mon Jun 5 09:54:29 2017 -0300
11557: Create can_read link on reused jobs
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at curoverse.com>
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 85ada26..c3eb919 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -292,9 +292,9 @@ class RunnerJob(Runner):
).execute(num_retries=self.arvrunner.num_retries)
if self.enable_reuse:
+ # When reusing jobs, copy its output/log collection to the desired project
reused_collections = [('Output', job['output']), ('Log', job['log'])]
for col_type, pdh in [(n, p) for n, p in reused_collections if p]:
- # When reusing jobs, copy its output/log collection to the desired project
c = arvados.collection.Collection(pdh,
api_client=self.arvrunner.api,
keep_client=self.arvrunner.keep_client,
@@ -306,6 +306,13 @@ class RunnerJob(Runner):
logger.info("Copied reused job's %s to collection %s",
col_type.lower(),
c.manifest_locator())
+ # Give read permission to the desired project on reused jobs
+ for job_name, job_uuid in job['components'].items():
+ self.arvrunner.api.links().create(body={
+ 'link_class': 'can_read',
+ 'tail_uuid': self.arvrunner.project_uuid,
+ 'head_uuid': job_uuid,
+ }).execute(num_retries=self.arvrunner.num_retries)
for k,v in job_spec["script_parameters"].items():
if v is False or v is None or isinstance(v, dict):
commit 3120003dc579730ac67cac8a47f209b14ec748d3
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Mon Jun 5 09:20:20 2017 -0300
11557: Added log collection copying to local project when reusing jobs
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at curoverse.com>
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index cf5a2ce..85ada26 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -291,17 +291,21 @@ class RunnerJob(Runner):
find_or_create=self.enable_reuse
).execute(num_retries=self.arvrunner.num_retries)
- if self.enable_reuse and job['output']:
- # When reusing jobs, copy its output collection to the desired project
- c = arvados.collection.Collection(job['output'],
- api_client=self.arvrunner.api,
- keep_client=self.arvrunner.keep_client,
- num_retries=self.arvrunner.num_retries)
- c.save_new(name="Output of {}".format(self.name),
- owner_uuid=self.arvrunner.project_uuid,
- ensure_unique_name=True,
- num_retries=self.arvrunner.num_retries)
- logger.info("Copied reused job's output to collection %s", c.manifest_locator())
+ if self.enable_reuse:
+ reused_collections = [('Output', job['output']), ('Log', job['log'])]
+ for col_type, pdh in [(n, p) for n, p in reused_collections if p]:
+ # When reusing jobs, copy its output/log collection to the desired project
+ c = arvados.collection.Collection(pdh,
+ api_client=self.arvrunner.api,
+ keep_client=self.arvrunner.keep_client,
+ num_retries=self.arvrunner.num_retries)
+ c.save_new(name="{} of {}".format(col_type, self.name),
+ owner_uuid=self.arvrunner.project_uuid,
+ ensure_unique_name=True,
+ num_retries=self.arvrunner.num_retries)
+ logger.info("Copied reused job's %s to collection %s",
+ col_type.lower(),
+ c.manifest_locator())
for k,v in job_spec["script_parameters"].items():
if v is False or v is None or isinstance(v, dict):
commit d516e20a5cfce7f1f0c78587f3ce847125c98921
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Thu Jun 1 23:15:37 2017 -0300
11557: Copy reused job's output to a collection on specified project (or Home project by default)
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at curoverse.com>
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 0bf91f2..cf5a2ce 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -291,6 +291,18 @@ class RunnerJob(Runner):
find_or_create=self.enable_reuse
).execute(num_retries=self.arvrunner.num_retries)
+ if self.enable_reuse and job['output']:
+ # When reusing jobs, copy its output collection to the desired project
+ c = arvados.collection.Collection(job['output'],
+ api_client=self.arvrunner.api,
+ keep_client=self.arvrunner.keep_client,
+ num_retries=self.arvrunner.num_retries)
+ c.save_new(name="Output of {}".format(self.name),
+ owner_uuid=self.arvrunner.project_uuid,
+ ensure_unique_name=True,
+ num_retries=self.arvrunner.num_retries)
+ logger.info("Copied reused job's output to collection %s", c.manifest_locator())
+
for k,v in job_spec["script_parameters"].items():
if v is False or v is None or isinstance(v, dict):
job_spec["script_parameters"][k] = {"value": v}
commit 42677f11798ac78f2898fe5ce9913ff8e14578e8
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Thu Jun 1 11:53:31 2017 -0300
11557: Simplifying conditional usage
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at curoverse.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 43082df..f7da563 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -449,8 +449,7 @@ class ArvCwlRunner(object):
name=kwargs.get("name"),
on_error=kwargs.get("on_error"),
submit_runner_image=kwargs.get("submit_runner_image"))
-
- if not kwargs.get("submit") and "cwl_runner_job" not in kwargs and self.work_api == "jobs":
+ elif "cwl_runner_job" not in kwargs and self.work_api == "jobs":
# Create pipeline for local run
self.pipeline = self.api.pipeline_instances().create(
body={
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list