[ARVADOS] created: 2.1.0-2402-g78d295050
Git user
git at public.arvados.org
Fri May 6 14:21:06 UTC 2022
at 78d295050b72f310a6c3ae24f856a12d02491509 (commit)
commit 78d295050b72f310a6c3ae24f856a12d02491509
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri May 6 10:20:47 2022 -0400
17004: Setting intermediate collection fields
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 4fbade074..1fd5ebf99 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -146,6 +146,8 @@ class ArvadosContainer(JobBase):
mounts[targetdir]["path"] = path
prevdir = targetdir + "/"
+ intermediate_collection_info = arvados_cwl.util.get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
+
with Perf(metrics, "generatefiles %s" % self.name):
if self.generatefiles["listing"]:
vwd = arvados.collection.Collection(api_client=self.arvrunner.api,
@@ -197,12 +199,11 @@ class ArvadosContainer(JobBase):
if not runtimeContext.current_container:
runtimeContext.current_container = arvados_cwl.util.get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
- info = arvados_cwl.util.get_intermediate_collection_info(self.name, runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
vwd.save_new(name=info["name"],
owner_uuid=runtimeContext.project_uuid,
ensure_unique_name=True,
- trash_at=info["trash_at"],
- properties=info["properties"])
+ trash_at=intermediate_collection_info["trash_at"],
+ properties=intermediate_collection_info["properties"])
prev = None
for f, p in sorteditems:
@@ -345,6 +346,8 @@ class ArvadosContainer(JobBase):
for pr in output_properties_req["processProperties"]:
container_request["output_properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+ output_properties.update(intermediate_collection_info["properties"])
+
if runtimeContext.runnerjob.startswith("arvwf:"):
wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")]
wfrecord = self.arvrunner.api.workflows().get(uuid=wfuuid).execute(num_retries=self.arvrunner.num_retries)
commit 774023a3d2953f22cfd5b85eb178d23b5ac11d03
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu May 5 16:00:37 2022 -0400
17004: Initial CWL support for setting output properties
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/api/methods/container_requests.html.textile.liquid b/doc/api/methods/container_requests.html.textile.liquid
index 870470110..cc5c19d79 100644
--- a/doc/api/methods/container_requests.html.textile.liquid
+++ b/doc/api/methods/container_requests.html.textile.liquid
@@ -61,6 +61,7 @@ table(table table-bordered table-condensed).
|runtime_user_uuid|string|The user permission that will be granted to this container.||
|runtime_auth_scopes|array of string|The scopes associated with the auth token used to run this container.||
|output_storage_classes|array of strings|The storage classes that will be used for the log and output collections of this container request|default is ["default"]|
+|output_properties|hash|User metadata properties to set on the output collection.|
h2(#priority). Priority
diff --git a/doc/api/methods/containers.html.textile.liquid b/doc/api/methods/containers.html.textile.liquid
index 18fb4f013..76e5730c9 100644
--- a/doc/api/methods/containers.html.textile.liquid
+++ b/doc/api/methods/containers.html.textile.liquid
@@ -60,6 +60,7 @@ Generally this will contain additional keys that are not present in any correspo
|gateway_address|string|Address (host:port) of gateway server.|Internal use only.|
|interactive_session_started|boolean|Indicates whether @arvados-client shell@ has been used to run commands in the container, which may have altered the container's behavior and output.||
|output_storage_classes|array of strings|The storage classes that will be used for the log and output collections of this container||
+|output_properties|hash|User metadata properties to set on the output collection.|
h2(#container_states). Container states
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index c73b358ec..67b80ff62 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -261,6 +261,7 @@ def add_arv_hints():
"http://arvados.org/cwl#ProcessProperties",
"http://commonwl.org/cwltool#CUDARequirement",
"http://arvados.org/cwl#UsePreemptible",
+ "http://arvados.org/cwl#OutputCollectionProperties",
])
def exit_signal_handler(sigcode, frame):
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
index af7548143..73b812097 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
@@ -299,8 +299,8 @@ $graph:
- type: record
name: PropertyDef
doc: |
- Define a property that will be set on the submitted container
- request associated with this workflow or step.
+ Define an arvados metadata property that will be set on a
+ container request or output collection.
fields:
- name: propertyName
type: string
@@ -400,3 +400,23 @@ $graph:
_id: "@type"
_type: "@vocab"
usePreemptible: boolean
+
+- name: OutputCollectionProperties
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify metadata properties that will be set on the output
+ collection associated with this workflow or step.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:OutputCollectionProperties"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ processProperties:
+ type: PropertyDef[]
+ jsonldPredicate:
+ mapSubject: propertyName
+ mapPredicate: propertyValue
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index 0ae451cca..2044dac2a 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -343,3 +343,23 @@ $graph:
_id: "@type"
_type: "@vocab"
usePreemptible: boolean
+
+- name: OutputCollectionProperties
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify metadata properties that will be set on the output
+ collection associated with this workflow or step.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:OutputCollectionProperties"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ processProperties:
+ type: PropertyDef[]
+ jsonldPredicate:
+ mapSubject: propertyName
+ mapPredicate: propertyValue
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index de5e55ca0..c969eef25 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -345,3 +345,23 @@ $graph:
_id: "@type"
_type: "@vocab"
usePreemptible: boolean
+
+- name: OutputCollectionProperties
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Specify metadata properties that will be set on the output
+ collection associated with this workflow or step.
+ fields:
+ class:
+ type: string
+ doc: "Always 'arv:OutputCollectionProperties"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ processProperties:
+ type: PropertyDef[]
+ jsonldPredicate:
+ mapSubject: propertyName
+ mapPredicate: propertyValue
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index f75bde81e..4fbade074 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -340,6 +340,11 @@ class ArvadosContainer(JobBase):
for pr in properties_req["processProperties"]:
container_request["properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+ output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties")
+ if output_properties_req:
+ for pr in output_properties_req["processProperties"]:
+ container_request["output_properties"][pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+
if runtimeContext.runnerjob.startswith("arvwf:"):
wfuuid = runtimeContext.runnerjob[6:runtimeContext.runnerjob.index("#")]
wfrecord = self.arvrunner.api.workflows().get(uuid=wfuuid).execute(num_retries=self.arvrunner.num_retries)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 680ca0b7b..47eedef80 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -404,7 +404,7 @@ The 'jobs' API is no longer supported.
with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
self.check_features(v, parentfield=parentfield)
- def make_output_collection(self, name, storage_classes, tagsString, outputObj):
+ def make_output_collection(self, name, storage_classes, tagsString, output_properties, outputObj):
outputObj = copy.deepcopy(outputObj)
files = []
@@ -456,7 +456,9 @@ The 'jobs' API is no longer supported.
res = str(json.dumps(outputObj, sort_keys=True, indent=4, separators=(',',': '), ensure_ascii=False))
f.write(res)
- final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes, ensure_unique_name=True)
+
+ final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes,
+ ensure_unique_name=True, properties=output_properties)
logger.info("Final output collection %s \"%s\" (%s)", final.portable_data_hash(),
final.api_response()["name"],
@@ -486,6 +488,7 @@ The 'jobs' API is no longer supported.
self.api.containers().update(uuid=current['uuid'],
body={
'output': self.final_output_collection.portable_data_hash(),
+ 'output_properties': self.final_output_collection.get_properties(),
}).execute(num_retries=self.num_retries)
self.api.collections().update(uuid=self.final_output_collection.manifest_locator(),
body={
@@ -772,7 +775,15 @@ The 'jobs' API is no longer supported.
else:
storage_classes = runtimeContext.storage_classes.strip().split(",")
- self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes, self.output_tags, self.final_output)
+ output_properties = {}
+ output_properties_req, _ = self.get_requirement("http://arvados.org/cwl#OutputCollectionProperties")
+ if output_properties_req:
+ for pr in output_properties_req["processProperties"]:
+ output_properties[pr["propertyName"]] = self.builder.do_eval(pr["propertyValue"])
+
+ self.final_output, self.final_output_collection = self.make_output_collection(self.output_name, storage_classes,
+ self.output_tags, output_properties,
+ self.final_output)
self.set_crunch_output()
if runtimeContext.compute_checksum:
commit 4c81f26bcdeea0663565a6bb77925adfd91f9de0
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu May 5 11:49:48 2022 -0400
17004: Initial API server support for output_properties
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index 0326b1298..eedf71691 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -23,6 +23,7 @@ class Container < ArvadosModel
attribute :runtime_status, :jsonbHash, default: {}
attribute :runtime_auth_scopes, :jsonbArray, default: []
attribute :output_storage_classes, :jsonbArray, default: lambda { Rails.configuration.DefaultStorageClasses }
+ attribute :output_properties, :jsonbHash, default: {}
serialize :environment, Hash
serialize :mounts, Hash
@@ -81,6 +82,7 @@ class Container < ArvadosModel
t.add :gateway_address
t.add :interactive_session_started
t.add :output_storage_classes
+ t.add :output_properties
end
# Supported states for a container
@@ -485,7 +487,7 @@ class Container < ArvadosModel
:runtime_constraints, :scheduling_parameters,
:secret_mounts, :runtime_token,
:runtime_user_uuid, :runtime_auth_scopes,
- :output_storage_classes)
+ :output_storage_classes, :output_properties)
end
case self.state
@@ -496,7 +498,7 @@ class Container < ArvadosModel
permitted.push :priority
when Running
- permitted.push :priority, *progress_attrs
+ permitted.push :priority, :output_properties, *progress_attrs
if self.state_changed?
permitted.push :started_at, :gateway_address
end
diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb
index bec3deb29..74fc0802d 100644
--- a/services/api/app/models/container_request.rb
+++ b/services/api/app/models/container_request.rb
@@ -24,6 +24,7 @@ class ContainerRequest < ArvadosModel
attribute :properties, :jsonbHash, default: {}
attribute :secret_mounts, :jsonbHash, default: {}
attribute :output_storage_classes, :jsonbArray, default: lambda { Rails.configuration.DefaultStorageClasses }
+ attribute :output_properties, :jsonbHash, default: {}
serialize :environment, Hash
serialize :mounts, Hash
@@ -78,6 +79,7 @@ class ContainerRequest < ArvadosModel
t.add :state
t.add :use_existing
t.add :output_storage_classes
+ t.add :output_properties
end
# Supported states for a container request
@@ -100,7 +102,7 @@ class ContainerRequest < ArvadosModel
:output_path, :priority, :runtime_token,
:runtime_constraints, :state, :container_uuid, :use_existing,
:scheduling_parameters, :secret_mounts, :output_name, :output_ttl,
- :output_storage_classes]
+ :output_storage_classes, :output_properties]
def self.any_preemptible_instances?
Rails.configuration.InstanceTypes.any? do |k, v|
@@ -222,11 +224,7 @@ class ContainerRequest < ArvadosModel
owner_uuid: self.owner_uuid,
name: coll_name,
manifest_text: "",
- storage_classes_desired: self.output_storage_classes,
- properties: {
- 'type' => out_type,
- 'container_request' => uuid,
- })
+ storage_classes_desired: self.output_storage_classes)
end
if out_type == "log"
@@ -238,11 +236,21 @@ class ContainerRequest < ArvadosModel
manifest = dst.manifest_text
end
+ merged_properties = {}
+ if out_type == "output"
+ merged_properties.update(container.output_properties)
+ merged_properties.update(self.output_properties)
+ end
+
+ merged_properties['type'] = out_type
+ merged_properties['container_request'] = uuid
+
coll.assign_attributes(
portable_data_hash: Digest::MD5.hexdigest(manifest) + '+' + manifest.bytesize.to_s,
manifest_text: manifest,
trash_at: trash_at,
- delete_at: trash_at)
+ delete_at: trash_at,
+ properties: merged_properties)
coll.save_with_unique_name!
self.send(out_type + '_uuid=', coll.uuid)
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list