[arvados] created: 2.6.0-583-gcab6238cf1
git repository hosting
git at public.arvados.org
Tue Sep 12 01:47:04 UTC 2023
at cab6238cf14dedb0d34bdb477dbb1bc5717e9c4e (commit)
commit cab6238cf14dedb0d34bdb477dbb1bc5717e9c4e
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Sep 11 21:44:48 2023 -0400
20825: Remove print statement
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index f5f666ec73..5dacdeb48b 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -783,8 +783,6 @@ class ArvadosWorkflow(Workflow):
if rpn:
name = builder.do_eval(rpn)
- print("BBB name", name)
-
return RunnerContainer(self.arvrunner,
self,
self.loadingContext,
commit c5cf93df9f29a383484040cba02099440f1a62d1
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Sep 11 21:38:13 2023 -0400
20825: Add runnerProcessName
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index 751c71bdf8..389add4104 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -444,3 +444,7 @@ $graph:
jsonldPredicate:
_id: "@type"
_type: "@vocab"
+ - name: runnerProcessName
+ type: ['null', string, cwl:Expression]
+ doc: |
+ Custom name to use for the runner process
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index f4194174f9..f5f666ec73 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -775,7 +775,16 @@ class ArvadosWorkflow(Workflow):
return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
- def separateRunner(self, joborder, output_callback, runtimeContext):
+ def separateRunner(self, joborder, output_callback, runtimeContext, req, builder):
+
+ name = runtimeContext.name
+
+ rpn = req.get("runnerProcessName")
+ if rpn:
+ name = builder.do_eval(rpn)
+
+ print("BBB name", name)
+
return RunnerContainer(self.arvrunner,
self,
self.loadingContext,
@@ -783,7 +792,7 @@ class ArvadosWorkflow(Workflow):
None,
None,
submit_runner_ram=runtimeContext.submit_runner_ram,
- name=runtimeContext.name,
+ name=name,
on_error=runtimeContext.on_error,
submit_runner_image=runtimeContext.submit_runner_image,
intermediate_output_ttl=runtimeContext.intermediate_output_ttl,
@@ -806,7 +815,7 @@ class ArvadosWorkflow(Workflow):
req, _ = self.get_requirement("http://arvados.org/cwl#SeparateRunner")
if req:
- return self.separateRunner(joborder, output_callback, runtimeContext)
+ return self.separateRunner(joborder, output_callback, runtimeContext, req, builder)
return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
commit 2fa22cdb88faab1d3895daa876cd0df0c9aaa145
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Sep 11 21:25:31 2023 -0400
20825: Fix edge case producing incorrect error
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index eaf26a5791..3bfb8a7961 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -876,7 +876,8 @@ The 'jobs' API is no longer supported.
if (self.task_queue.in_flight + len(self.processes)) > 0:
self.workflow_eval_lock.wait(3)
else:
- logger.error("Workflow is deadlocked, no runnable processes and not waiting on any pending processes.")
+ if self.final_status is None:
+ logger.error("Workflow is deadlocked, no runnable processes and not waiting on any pending processes.")
break
if self.stop_polling.is_set():
commit 50a31136c0203a4f3eeea71ef4677db4fdeef7c8
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Sep 11 21:09:14 2023 -0400
20825: Fix git_info
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 11ae66b139..76a882a083 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -617,6 +617,8 @@ class RunnerContainer(Runner):
"content": packed
}
container_req["properties"]["template_uuid"] = self.embedded_tool.tool["id"][6:33]
+ elif self.embedded_tool.tool.get("id", "").startswith("file:"):
+ raise Exception("Tool id '%s' is a local file but expected keep: or arvwf:" % self.embedded_tool.tool.get("id"))
else:
main = self.loadingContext.loader.idx["_:main"]
if main.get("id") == "_:main":
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 0ef30b6c6d..f4194174f9 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -792,7 +792,7 @@ class ArvadosWorkflow(Workflow):
secret_store=self.arvrunner.secret_store,
collection_cache_size=runtimeContext.collection_cache_size,
collection_cache_is_default=self.arvrunner.should_estimate_cache_size,
- git_info=self.arvrunner.git_info).job(joborder, output_callback, runtimeContext)
+ git_info=runtimeContext.git_info).job(joborder, output_callback, runtimeContext)
def job(self, joborder, output_callback, runtimeContext):
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index dd64879b9f..0439cb5b15 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -45,6 +45,7 @@ class ArvRuntimeContext(RuntimeContext):
self.prefer_cached_downloads = False
self.cached_docker_lookups = {}
self.print_keep_deps = False
+ self.git_info = {}
super(ArvRuntimeContext, self).__init__(kwargs)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 330dba3dbe..eaf26a5791 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -603,6 +603,8 @@ The 'jobs' API is no longer supported.
if git_info[g]:
logger.info(" %s: %s", g.split("#", 1)[1], git_info[g])
+ runtimeContext.git_info = git_info
+
workbench1 = self.api.config()["Services"]["Workbench1"]["ExternalURL"]
workbench2 = self.api.config()["Services"]["Workbench2"]["ExternalURL"]
controller = self.api.config()["Services"]["Controller"]["ExternalURL"]
commit 08a45400540e9e5241a7f754d8395310eac080bf
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Sep 11 20:58:18 2023 -0400
20825: Fixing invocation
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 7230f25378..0ef30b6c6d 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -38,6 +38,7 @@ import ruamel.yaml as yaml
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
make_builder, arvados_jobs_image, FileUpdates)
+from .arvcontainer import RunnerContainer
from .pathmapper import ArvPathMapper, trim_listing
from .arvtool import ArvadosCommandTool, set_cluster_target
from ._version import __version__
@@ -774,11 +775,13 @@ class ArvadosWorkflow(Workflow):
return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
- def separateRunner(joborder, output_callback, runtimeContext, builder):
- return RunnerContainer(self, self.tool, self.loadingContext,
+ def separateRunner(self, joborder, output_callback, runtimeContext):
+ return RunnerContainer(self.arvrunner,
+ self,
+ self.loadingContext,
runtimeContext.enable_reuse,
- self.output_name,
- self.output_tags,
+ None,
+ None,
submit_runner_ram=runtimeContext.submit_runner_ram,
name=runtimeContext.name,
on_error=runtimeContext.on_error,
@@ -788,8 +791,8 @@ class ArvadosWorkflow(Workflow):
priority=runtimeContext.priority,
secret_store=self.arvrunner.secret_store,
collection_cache_size=runtimeContext.collection_cache_size,
- collection_cache_is_default=self.should_estimate_cache_size,
- git_info=self.arvrunner.git_info)
+ collection_cache_is_default=self.arvrunner.should_estimate_cache_size,
+ git_info=self.arvrunner.git_info).job(joborder, output_callback, runtimeContext)
def job(self, joborder, output_callback, runtimeContext):
@@ -803,7 +806,7 @@ class ArvadosWorkflow(Workflow):
req, _ = self.get_requirement("http://arvados.org/cwl#SeparateRunner")
if req:
- return self.separateRunner(joborder, output_callback, runtimeContext, builder)
+ return self.separateRunner(joborder, output_callback, runtimeContext)
return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
commit bcb6992c7ae2d577cdb97aca67e4bcd96df90683
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Sep 11 20:42:55 2023 -0400
20825: arv:SeparateRunner first try
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index f4246ed70a..751c71bdf8 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -429,3 +429,18 @@ $graph:
doc: |
If the container failed on its first run, re-submit the
container with the RAM request multiplied by this factor.
+
+- name: SeparateRunner
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Indicates that a subworkflow should run in a separate
+ arvados-cwl-runner process.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:SeparateRunner'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index cdce3d643a..7230f25378 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -609,17 +609,8 @@ class ArvadosWorkflow(Workflow):
super(ArvadosWorkflow, self).__init__(toolpath_object, self.loadingContext)
self.cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
- def job(self, joborder, output_callback, runtimeContext):
-
- builder = make_builder(joborder, self.hints, self.requirements, runtimeContext, self.metadata)
- runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
-
- req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
- if not req:
- return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
-
- # RunInSingleContainer is true
+ def runInSingleContainer(self, joborder, output_callback, runtimeContext, builder):
with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
if "id" not in self.tool:
raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
@@ -782,6 +773,41 @@ class ArvadosWorkflow(Workflow):
})
return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
+
+ def separateRunner(joborder, output_callback, runtimeContext, builder):
+ return RunnerContainer(self, self.tool, self.loadingContext,
+ runtimeContext.enable_reuse,
+ self.output_name,
+ self.output_tags,
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=runtimeContext.name,
+ on_error=runtimeContext.on_error,
+ submit_runner_image=runtimeContext.submit_runner_image,
+ intermediate_output_ttl=runtimeContext.intermediate_output_ttl,
+ merged_map=None,
+ priority=runtimeContext.priority,
+ secret_store=self.arvrunner.secret_store,
+ collection_cache_size=runtimeContext.collection_cache_size,
+ collection_cache_is_default=self.should_estimate_cache_size,
+ git_info=self.arvrunner.git_info)
+
+
+ def job(self, joborder, output_callback, runtimeContext):
+
+ builder = make_builder(joborder, self.hints, self.requirements, runtimeContext, self.metadata)
+ runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
+
+ req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
+ if req:
+ return self.runInSingleContainer(joborder, output_callback, runtimeContext, builder)
+
+ req, _ = self.get_requirement("http://arvados.org/cwl#SeparateRunner")
+ if req:
+ return self.separateRunner(joborder, output_callback, runtimeContext, builder)
+
+ return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
+
+
def make_workflow_step(self,
toolpath_object, # type: Dict[Text, Any]
pos, # type: int
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index a93c64a224..e0bdd8a5a3 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -494,3 +494,9 @@
output: {}
tool: oom/19975-oom3.cwl
doc: "Test feature 19975 - retry on custom error"
+
+- job: null
+ output:
+ out: out
+ tool: wf/runseparate-wf.cwl
+ doc: "test arv:SeparateRunner"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list