[ARVADOS] updated: 1.3.0-3037-gb36ffab02
Git user
git at public.arvados.org
Fri Aug 28 20:05:28 UTC 2020
Summary of changes:
doc/install/arvbox.html.textile.liquid | 11 ++++
...quid => arvados-cwl-runner.html.textile.liquid} | 34 ++++++++++-
lib/cloud/azure/azure.go | 4 +-
lib/cloud/azure/azure_test.go | 4 +-
lib/cloud/ec2/ec2.go | 8 +--
lib/cloud/ec2/ec2_test.go | 8 +--
lib/controller/handler_test.go | 4 +-
lib/controller/localdb/login_testuser.go | 2 +-
lib/dispatchcloud/scheduler/run_queue_test.go | 2 +-
lib/dispatchcloud/test/queue.go | 6 +-
.../api/app/models/api_client_authorization.rb | 2 +-
services/api/app/models/database_seeds.rb | 1 +
services/api/lib/current_api_client.rb | 10 ++++
services/api/test/fixtures/api_clients.yml | 7 +++
.../api_client_authorizations_api_test.rb | 46 +++++++++-----
services/arv-git-httpd/auth_handler_test.go | 2 +-
services/arv-git-httpd/git_handler_test.go | 2 +-
services/arv-git-httpd/integration_test.go | 2 +-
services/keepproxy/keepproxy_test.go | 2 +-
services/keepstore/proxy_remote_test.go | 2 +-
services/ws/service_test.go | 2 +-
tools/arvbox/bin/arvbox | 21 ++++++-
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 2 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.demo | 4 +-
tools/arvbox/lib/arvbox/docker/cluster-config.sh | 23 +++----
tools/arvbox/lib/arvbox/docker/edit_users.py | 70 ++++++++++++++++++++++
26 files changed, 224 insertions(+), 57 deletions(-)
copy doc/sdk/python/{arvados-fuse.html.textile.liquid => arvados-cwl-runner.html.textile.liquid} (59%)
create mode 100755 tools/arvbox/lib/arvbox/docker/edit_users.py
discards db251a6bdf0179ebaf820ac3a06a150f2a3b7e1d (commit)
discards 940ca546c8150200c64e44f0ecbc30c9d4e59bc5 (commit)
discards c502d965f4dbc07aa3b9ed2ea233872c3c6a0b8e (commit)
discards 4ddeddd8d58e502dd471213198f421c842e7a5c7 (commit)
discards 18e2ef8679a7c1a3e1d9121d308e9b94fa7573e3 (commit)
via b36ffab0228e53226614f7d33e4a8e3921d0256f (commit)
via ae50c333f57298b5d4b81229476cd990aab1dad5 (commit)
via ecc2f49233c0b52820e856ba0b18e4123d99d228 (commit)
via 434dee9ff1b42d7169fbd9368263e6a0f5c40bed (commit)
via 5ce67dcb90e196227d920c52fc1a7256e39ede92 (commit)
via d3b6bdd6ef2b543c607bd7c3cdf9df5c74e90dc0 (commit)
via be28c5f528a93ee32eef4c1dc2d0872cb718b29f (commit)
via 4706c89f091563cc56a6d4f819e025850031a009 (commit)
via 43da0e7ce859cb8ed3385417a2fc97a36cc688e6 (commit)
via ecff7e34ea7d5e8321c2821cae476355bffbc248 (commit)
via 6d827e6be83d9b3129b4bc7a812d43d2ca874174 (commit)
via d67b634b9afe9bebeaef461dfdd2edfa4e5740fd (commit)
via 5895b2710b4331109a0056275e8b046a53b5ba06 (commit)
via 1617202c337078fb94ea19893c73061983be94ad (commit)
via 3e975aa25c141ccd3f08335906d96d9ff7035bf2 (commit)
via 81ff58f4addd05346161a9b44648d1ab31e027bc (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (db251a6bdf0179ebaf820ac3a06a150f2a3b7e1d)
\
N -- N -- N (b36ffab0228e53226614f7d33e4a8e3921d0256f)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b36ffab0228e53226614f7d33e4a8e3921d0256f
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Aug 28 16:04:19 2020 -0400
16602: Tweak fixture 'name' name to fix test
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/api/test/fixtures/workflows.yml b/services/api/test/fixtures/workflows.yml
index 98bfeaf84..29b76abb4 100644
--- a/services/api/test/fixtures/workflows.yml
+++ b/services/api/test/fixtures/workflows.yml
@@ -71,8 +71,8 @@ workflow_with_input_defaults:
workflow_with_wrr:
uuid: zzzzz-7fd4e-validwithinput3
owner_uuid: zzzzz-j7d0g-zhxawtyetzwc5f0
- name: Workflow with default input specifications
- description: this workflow has inputs specified
+ name: Workflow with WorkflowRunnerResources
+ description: this workflow has WorkflowRunnerResources
created_at: <%= 1.minute.ago.to_s(:db) %>
definition: |
cwlVersion: v1.0
commit ae50c333f57298b5d4b81229476cd990aab1dad5
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Aug 27 21:38:34 2020 -0400
16602: Test for applying WorkflowRunnerResource
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/apps/workbench/test/integration/work_units_test.rb b/apps/workbench/test/integration/work_units_test.rb
index 9d4f59055..43740f192 100644
--- a/apps/workbench/test/integration/work_units_test.rb
+++ b/apps/workbench/test/integration/work_units_test.rb
@@ -283,4 +283,23 @@ class WorkUnitsTest < ActionDispatch::IntegrationTest
assert_text "This container request was created from the workflow"
assert_match /Provide a value for .* then click the \"Run\" button to start the workflow/, page.text
end
+
+ test "create workflow with WorkflowRunnerResources" do
+ visit page_with_token('active', '/workflows/zzzzz-7fd4e-validwithinput3')
+
+ find('a,button', text: 'Run this workflow').click
+
+ # Choose project for the container_request being created
+ within('.modal-dialog') do
+ find('.selectable', text: 'A Project').click
+ find('button', text: 'Choose').click
+ end
+ click_link 'Advanced'
+ click_link("API response")
+ assert_text('"container_image": "arvados/jobs:2.0.4"')
+ assert_text('"vcpus": 2')
+ assert_text('"ram": 1293942784')
+ assert_text('"--collection-cache-size=678"')
+
+ end
end
diff --git a/services/api/test/fixtures/workflows.yml b/services/api/test/fixtures/workflows.yml
index 2859e375a..98bfeaf84 100644
--- a/services/api/test/fixtures/workflows.yml
+++ b/services/api/test/fixtures/workflows.yml
@@ -67,3 +67,28 @@ workflow_with_input_defaults:
id: ex_string_def
default: hello-testing-123
outputs: []
+
+workflow_with_wrr:
+ uuid: zzzzz-7fd4e-validwithinput3
+ owner_uuid: zzzzz-j7d0g-zhxawtyetzwc5f0
+ name: Workflow with default input specifications
+ description: this workflow has inputs specified
+ created_at: <%= 1.minute.ago.to_s(:db) %>
+ definition: |
+ cwlVersion: v1.0
+ class: CommandLineTool
+ hints:
+ - class: http://arvados.org/cwl#WorkflowRunnerResources
+ acrContainerImage: arvados/jobs:2.0.4
+ ramMin: 1234
+ coresMin: 2
+ keep_cache: 678
+ baseCommand:
+ - echo
+ inputs:
+ - type: string
+ id: ex_string
+ - type: string
+ id: ex_string_def
+ default: hello-testing-123
+ outputs: []
commit ecc2f49233c0b52820e856ba0b18e4123d99d228
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Aug 26 17:22:44 2020 -0400
16602: Add missing file
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/tests/wf/expect_upload_packed.cwl b/sdk/cwl/tests/wf/expect_upload_packed.cwl
new file mode 100644
index 000000000..9a50fc8ab
--- /dev/null
+++ b/sdk/cwl/tests/wf/expect_upload_packed.cwl
@@ -0,0 +1,100 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{
+ "$graph": [
+ {
+ "baseCommand": "cat",
+ "class": "CommandLineTool",
+ "id": "#submit_tool.cwl",
+ "inputs": [
+ {
+ "default": {
+ "class": "File",
+ "location": "keep:5d373e7629203ce39e7c22af98a0f881+52/blub.txt"
+ },
+ "id": "#submit_tool.cwl/x",
+ "inputBinding": {
+ "position": 1
+ },
+ "type": "File"
+ }
+ ],
+ "outputs": [],
+ "requirements": [
+ {
+ "class": "DockerRequirement",
+ "dockerPull": "debian:8",
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
+ }
+ ]
+ },
+ {
+ "class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
+ "id": "#main",
+ "inputs": [
+ {
+ "default": {
+ "basename": "blorp.txt",
+ "class": "File",
+ "location": "keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt",
+ "nameext": ".txt",
+ "nameroot": "blorp",
+ "size": 16
+ },
+ "id": "#main/x",
+ "type": "File"
+ },
+ {
+ "default": {
+ "basename": "99999999999999999999999999999998+99",
+ "class": "Directory",
+ "location": "keep:99999999999999999999999999999998+99"
+ },
+ "id": "#main/y",
+ "type": "Directory"
+ },
+ {
+ "default": {
+ "basename": "anonymous",
+ "class": "Directory",
+ "listing": [
+ {
+ "basename": "renamed.txt",
+ "class": "File",
+ "location": "keep:99999999999999999999999999999998+99/file1.txt",
+ "nameext": ".txt",
+ "nameroot": "renamed",
+ "size": 0
+ }
+ ]
+ },
+ "id": "#main/z",
+ "type": "Directory"
+ }
+ ],
+ "outputs": [],
+ "steps": [
+ {
+ "id": "#main/step1",
+ "in": [
+ {
+ "id": "#main/step1/x",
+ "source": "#main/x"
+ }
+ ],
+ "out": [],
+ "run": "#submit_tool.cwl"
+ }
+ ]
+ }
+ ],
+ "cwlVersion": "v1.0"
+}
commit 434dee9ff1b42d7169fbd9368263e6a0f5c40bed
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Aug 26 15:44:59 2020 -0400
16602: Workbench sets container_image from acrContainerImage
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/apps/workbench/app/controllers/work_units_controller.rb b/apps/workbench/app/controllers/work_units_controller.rb
index 8c4e5e7d9..ba2f66abe 100644
--- a/apps/workbench/app/controllers/work_units_controller.rb
+++ b/apps/workbench/app/controllers/work_units_controller.rb
@@ -117,6 +117,9 @@ class WorkUnitsController < ApplicationController
if hint[:keep_cache]
keep_cache = hint[:keep_cache]
end
+ if hint[:acrContainerImage]
+ attrs['container_image'] = hint[:acrContainerImage]
+ end
end
end
end
commit 5ce67dcb90e196227d920c52fc1a7256e39ede92
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Aug 26 15:40:33 2020 -0400
16602: Embed the desired workflow runner version in WorkflowRunnerResources
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
index dce1bd4d0..8a3fa3173 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
@@ -240,6 +240,12 @@ $graph:
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index b9b9e6165..95ed0a75b 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -184,6 +184,12 @@ $graph:
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index b9b9e6165..95ed0a75b 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -184,6 +184,12 @@ $graph:
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 97c5fafe7..56c6f39e9 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -24,9 +24,10 @@ import ruamel.yaml as yaml
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
- make_builder)
+ make_builder, arvados_jobs_image)
from .pathmapper import ArvPathMapper, trim_listing
from .arvtool import ArvadosCommandTool, set_cluster_target
+from ._version import __version__
from .perf import Perf
@@ -37,7 +38,8 @@ max_res_pars = ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdir
sum_res_pars = ("outdirMin", "outdirMax")
def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
- submit_runner_ram=0, name=None, merged_map=None):
+ submit_runner_ram=0, name=None, merged_map=None,
+ submit_runner_image=None):
packed = packed_workflow(arvRunner, tool, merged_map)
@@ -57,18 +59,25 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
upload_dependencies(arvRunner, name, tool.doc_loader,
packed, tool.tool["id"], False)
+ wf_runner_resources = None
+
+ hints = main.get("hints", [])
+ found = False
+ for h in hints:
+ if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
+ wf_runner_resources = h
+ found = True
+ break
+ if not found:
+ wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"}
+ hints.append(wf_runner_resources)
+
+ wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, submit_runner_image or "arvados/jobs:"+__version__)
+
if submit_runner_ram:
- hints = main.get("hints", [])
- found = False
- for h in hints:
- if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
- h["ramMin"] = submit_runner_ram
- found = True
- break
- if not found:
- hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources",
- "ramMin": submit_runner_ram})
- main["hints"] = hints
+ wf_runner_resources["ramMin"] = submit_runner_ram
+
+ main["hints"] = hints
body = {
"workflow": {
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index e8d1347dd..9ba798ec6 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -596,7 +596,8 @@ The 'jobs' API is no longer supported.
uuid=existing_uuid,
submit_runner_ram=runtimeContext.submit_runner_ram,
name=runtimeContext.name,
- merged_map=merged_map),
+ merged_map=merged_map,
+ submit_runner_image=runtimeContext.submit_runner_image),
"success")
self.apply_reqs(job_order, tool)
diff --git a/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl b/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
index 9bf1c20aa..1054d8f29 100644
--- a/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
+++ b/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
@@ -6,6 +6,12 @@
"$graph": [
{
"class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
"id": "#main",
"inputs": [],
"outputs": [],
@@ -82,4 +88,4 @@
}
],
"cwlVersion": "v1.0"
-}
\ No newline at end of file
+}
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 0698db70f..4da545bf3 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1354,7 +1354,7 @@ class TestSubmit(unittest.TestCase):
class TestCreateWorkflow(unittest.TestCase):
existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
expect_workflow = StripYAMLComments(
- open("tests/wf/expect_packed.cwl").read())
+ open("tests/wf/expect_upload_packed.cwl").read().rstrip())
@stubs
def test_create(self, stubs):
@@ -1472,7 +1472,7 @@ class TestCreateWorkflow(unittest.TestCase):
stubs.capture_stdout, sys.stderr, api_client=stubs.api)
toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
- expect_workflow = StripYAMLComments(open(toolfile).read())
+ expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
body = {
"workflow": {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list