[ARVADOS] updated: 2.1.0-1948-gaa1020931
Git user
git at public.arvados.org
Fri Feb 18 16:57:24 UTC 2022
Summary of changes:
.../ubuntu1804/Dockerfile | 2 +-
build/run-build-packages.sh | 10 +-
build/run-library.sh | 2 +-
doc/_config.yml | 7 +-
doc/admin/config-migration.html.textile.liquid | 106 ----------
doc/admin/config.html.textile.liquid | 2 -
.../maintenance-and-upgrading.html.textile.liquid | 64 ++++++
doc/admin/upgrading.html.textile.liquid | 29 +--
doc/install/salt-multi-host.html.textile.liquid | 9 +-
doc/install/salt-single-host.html.textile.liquid | 7 +
lib/crunchrun/crunchrun_test.go | 8 +
lib/install/deps.go | 2 +-
lib/mount/fs.go | 23 +-
sdk/cwl/setup.py | 2 +-
sdk/cwl/tests/test_container.py | 91 ++++++++
sdk/cwl/tests/test_submit.py | 8 +-
sdk/go/arvados/fs_base.go | 63 +++++-
sdk/go/arvados/fs_collection.go | 129 ++++++++++++
sdk/go/arvados/fs_deferred.go | 2 +
sdk/go/arvados/fs_filehandle.go | 15 ++
sdk/go/arvados/fs_getternode.go | 2 +-
sdk/go/arvados/fs_lookup.go | 6 +-
sdk/go/arvados/fs_project_test.go | 29 ++-
sdk/go/arvados/fs_site.go | 6 +-
sdk/go/arvados/fs_site_test.go | 232 ++++++++++++++++++++-
services/keep-web/s3.go | 2 +-
services/keep-web/s3_test.go | 2 +-
tools/arvbox/lib/arvbox/docker/api-setup.sh | 6 +-
tools/arvbox/lib/arvbox/docker/common.sh | 2 +-
.../lib/arvbox/docker/service/doc/run-service | 2 +-
.../lib/arvbox/docker/service/gitolite/run-service | 2 +-
.../arvbox/docker/service/workbench/run-service | 6 +-
32 files changed, 701 insertions(+), 177 deletions(-)
delete mode 100644 doc/admin/config-migration.html.textile.liquid
create mode 100644 doc/admin/maintenance-and-upgrading.html.textile.liquid
discards 6d728a9567eda6e256359c3606a5de41f126a59f (commit)
via aa102093124b75fee1a71571eb1ab7545652827f (commit)
via 61b747425ea233ef9b91575a17aa2fabc5965def (commit)
via 92ee058f2beb6876348f6373f4fb9be72a56bcf7 (commit)
via 4752421d3d4b3a0f6afe93ce3356961d1d81b494 (commit)
via d571b37981a225d93ded87d042aef652ba04afe2 (commit)
via a9ffac64364fafe598c1e6af0468b9139cdd8083 (commit)
via a66462de44bf709365aaa562570c9b6b68dd92b7 (commit)
via f2388f1bdad27efd2816533aa7da80735ed5ec3f (commit)
via ca0b6ff3d75f68b7a2f1821d605fadd49481038e (commit)
via 70d4cba7d4aef3063fb549a77b45951c339cc57c (commit)
via 09c022661e08acbb560b4969f1127012d987b94e (commit)
via d4e10c0482b28a1c2fb7ef48e69a673d2c6ea15a (commit)
via 1fc3e6081d1d016184f328af84c1aaf330920be0 (commit)
via 438c39ee7bef019d33e90a6de9313381abacae68 (commit)
via be278ca4fbb020c06c6f6168ebc8a5adfe161307 (commit)
via c5c94eebf3fd8690ce32f98d9950fd9efcfbc67f (commit)
via 3e2c6bb988d3819fadb15f6446fbdfee0046600d (commit)
via b14d0a0ac78eac7d074ca81411d456605f61bb24 (commit)
via 668e4f412e70817059fa093b4739c06c50705e79 (commit)
via 3c3d72f3076f164c79a10e089b80a098ecec5ff8 (commit)
via a3c509e8eca36f1291f7547999f16d9fd127c4a0 (commit)
via 23bad0a705b1809a73ffbd5f6866e14dde5dd52e (commit)
via 3ad4e88f87ab4943be712a82d1d8269657b41f8a (commit)
via 846e3037de341d73e593a670b0d0e77bc3e893c1 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (6d728a9567eda6e256359c3606a5de41f126a59f)
\
N -- N -- N (aa102093124b75fee1a71571eb1ab7545652827f)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit aa102093124b75fee1a71571eb1ab7545652827f
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Feb 18 11:57:00 2022 -0500
18773: Add test for match_local_docker behavior
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 21ae4a7af..72774daba 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -1117,6 +1117,97 @@ class TestContainer(unittest.TestCase):
}))
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ @mock.patch("arvados_cwl.arvdocker.determine_image_id")
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ def test_match_local_docker(self, keepdocker, determine_image_id):
+ arvados_cwl.add_arv_hints()
+ arv_docker_clear_cache()
+
+ runner = mock.MagicMock()
+ runner.ignore_docker_for_reuse = False
+ runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+ runner.api._rootDesc = {"revision": "20210628"}
+
+ keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
+ ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
+ determine_image_id.side_effect = lambda x: "123"
+ def execute(uuid):
+ ex = mock.MagicMock()
+ lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"},
+ "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}}
+ ex.execute.return_value = lookup[uuid]
+ return ex
+ runner.api.collections().get.side_effect = execute
+
+ tool = cmap({
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "echo",
+ "arguments": [],
+ "id": "",
+ "cwlVersion": "v1.2",
+ "class": "CommandLineTool"
+ })
+
+ loadingContext, runtimeContext = self.helper(runner, True)
+
+ arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+ arvtool.formatgraph = None
+
+ container_request = {
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_run_True',
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 268435456
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'output_name': 'Output for step test_run_True',
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': '99999999999999999999999999999994+99',
+ 'command': ['echo'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {},
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
+ }
+
+ runtimeContext.match_local_docker = False
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(container_request))
+
+ arv_docker_clear_cache()
+ runtimeContext.match_local_docker = True
+ container_request['container_image'] = '99999999999999999999999999999993+99'
+ container_request['name'] = 'test_run_True_2'
+ container_request['output_name'] = 'Output for step test_run_True_2'
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(container_request))
+
+
+
class TestWorkflow(unittest.TestCase):
def setUp(self):
cwltool.process._names = set()
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 8a2aa7e34..ce5e8c55c 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1089,25 +1089,6 @@ class TestSubmit(unittest.TestCase):
arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
- @stubs
- def test_match_local_docker(self, stubs):
- stubs.docker_images["debian:buster-slim"] = [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {"dockerhash": "456"}),
- ("zzzzz-4zz18-zzzzzzzzzzzzzd5", {"dockerhash": "123"})]
-
- exited = arvados_cwl.main(
- ["--submit", "--no-wait", "--api=containers", "--debug",
- "tests/tool/submit_tool.cwl", "tests/submit_test_job.json"],
- stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-
- expect_container = {
- }
-
- stubs.api.container_requests().create.assert_called_with(
- body=JsonDiffMatcher(expect_container))
- self.assertEqual(stubs.capture_stdout.getvalue(),
- stubs.expect_container_request_uuid + '\n')
- self.assertEqual(exited, 0)
-
@stubs
def test_submit_secrets(self, stubs):
exited = arvados_cwl.main(
commit 61b747425ea233ef9b91575a17aa2fabc5965def
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Feb 17 15:31:17 2022 -0500
18773: Test WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 77f70851e..8a2aa7e34 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -67,10 +67,10 @@ def stubs(func):
stubs.keep_client = keep_client2
stubs.docker_images = {
- "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
- "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
- "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
- "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
+ "arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", {})],
+ "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", {})],
+ "arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", {})],
+ "arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {})],
}
def kd(a, b, image_name=None, image_tag=None):
return stubs.docker_images.get("%s:%s" % (image_name, image_tag), [])
@@ -1089,6 +1089,25 @@ class TestSubmit(unittest.TestCase):
arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
+ @stubs
+ def test_match_local_docker(self, stubs):
+ stubs.docker_images["debian:buster-slim"] = [("zzzzz-4zz18-zzzzzzzzzzzzzd6", {"dockerhash": "456"}),
+ ("zzzzz-4zz18-zzzzzzzzzzzzzd5", {"dockerhash": "123"})]
+
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/tool/submit_tool.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+ expect_container = {
+ }
+
+ stubs.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher(expect_container))
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ stubs.expect_container_request_uuid + '\n')
+ self.assertEqual(exited, 0)
+
@stubs
def test_submit_secrets(self, stubs):
exited = arvados_cwl.main(
commit 92ee058f2beb6876348f6373f4fb9be72a56bcf7
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Feb 17 14:37:51 2022 -0500
18773: --match-submitter-images to compare the local docker image with Arvados
Consults local list of Docker images to find the image id and chooses
that exact one, uploading it if necessary.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 734945c0f..826467cc0 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -152,6 +152,10 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help="When invoked with --submit --wait, always submit a runner to manage the workflow, even when only running a single CommandLineTool",
default=False)
+ parser.add_argument("--match-submitter-images", action="store_true",
+ default=False, dest="match_local_docker",
+ help="Where Arvados has more than one Docker image of the same name, use image from the Docker instance on the submitting node.")
+
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--submit-request-uuid",
default=None,
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 3c7e9cfaa..753c2c250 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -246,7 +246,8 @@ class ArvadosContainer(JobBase):
runtimeContext.pull_image,
runtimeContext.project_uuid,
runtimeContext.force_docker_pull,
- runtimeContext.tmp_outdir_prefix)
+ runtimeContext.tmp_outdir_prefix,
+ runtimeContext.match_local_docker)
network_req, _ = self.get_requirement("NetworkAccess")
if network_req:
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index 26408317c..04e2a4cff 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -6,6 +6,8 @@ import logging
import sys
import threading
import copy
+import re
+import subprocess
from schema_salad.sourceline import SourceLine
@@ -18,8 +20,44 @@ logger = logging.getLogger('arvados.cwl-runner')
cached_lookups = {}
cached_lookups_lock = threading.Lock()
+def determine_image_id(dockerImageId):
+ for line in (
+ subprocess.check_output( # nosec
+ ["docker", "images", "--no-trunc", "--all"]
+ )
+ .decode("utf-8")
+ .splitlines()
+ ):
+ try:
+ match = re.match(r"^([^ ]+)\s+([^ ]+)\s+([^ ]+)", line)
+ split = dockerImageId.split(":")
+ if len(split) == 1:
+ split.append("latest")
+ elif len(split) == 2:
+ # if split[1] doesn't match valid tag names, it is a part of repository
+ if not re.match(r"[\w][\w.-]{0,127}", split[1]):
+ split[0] = split[0] + ":" + split[1]
+ split[1] = "latest"
+ elif len(split) == 3:
+ if re.match(r"[\w][\w.-]{0,127}", split[2]):
+ split[0] = split[0] + ":" + split[1]
+ split[1] = split[2]
+ del split[2]
+
+ # check for repository:tag match or image id match
+ if match and (
+ (split[0] == match.group(1) and split[1] == match.group(2))
+ or dockerImageId == match.group(3)
+ ):
+ return match.group(3)
+ except ValueError:
+ pass
+
+ return None
+
+
def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid,
- force_pull, tmp_outdir_prefix):
+ force_pull, tmp_outdir_prefix, match_local_docker):
"""Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement:
@@ -46,6 +84,20 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
image_name=image_name,
image_tag=image_tag)
+ if images and match_local_docker:
+ local_image_id = determine_image_id(dockerRequirement["dockerImageId"])
+ if local_image_id:
+ # find it in the list
+ found = False
+ for i in images:
+ if i[1]["dockerhash"] == local_image_id:
+ found = True
+ images = [i]
+ break
+ if not found:
+ # force re-upload.
+ images = []
+
if not images:
# Fetch Docker image if necessary.
try:
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 1e04dd577..4239dd3b5 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -36,6 +36,7 @@ class ArvRuntimeContext(RuntimeContext):
self.cluster_target_id = 0
self.always_submit_runner = False
self.collection_cache_size = 256
+ self.match_local_docker = False
super(ArvRuntimeContext, self).__init__(kwargs)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 7d6d287a2..ad17950a2 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -461,12 +461,14 @@ def upload_docker(arvrunner, tool):
"Option 'dockerOutputDirectory' of DockerRequirement not supported.")
arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid,
arvrunner.runtimeContext.force_docker_pull,
- arvrunner.runtimeContext.tmp_outdir_prefix)
+ arvrunner.runtimeContext.tmp_outdir_prefix,
+ arvrunner.runtimeContext.match_local_docker)
else:
arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs:"+__version__},
True, arvrunner.project_uuid,
arvrunner.runtimeContext.force_docker_pull,
- arvrunner.runtimeContext.tmp_outdir_prefix)
+ arvrunner.runtimeContext.tmp_outdir_prefix,
+ arvrunner.runtimeContext.match_local_docker)
elif isinstance(tool, cwltool.workflow.Workflow):
for s in tool.steps:
upload_docker(arvrunner, s.embedded_tool)
@@ -503,7 +505,8 @@ def packed_workflow(arvrunner, tool, merged_map):
v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True,
arvrunner.project_uuid,
arvrunner.runtimeContext.force_docker_pull,
- arvrunner.runtimeContext.tmp_outdir_prefix)
+ arvrunner.runtimeContext.tmp_outdir_prefix,
+ arvrunner.runtimeContext.match_local_docker)
for l in v:
visit(v[l], cur_id)
if isinstance(v, list):
@@ -610,7 +613,8 @@ def arvados_jobs_image(arvrunner, img):
try:
return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid,
arvrunner.runtimeContext.force_docker_pull,
- arvrunner.runtimeContext.tmp_outdir_prefix)
+ arvrunner.runtimeContext.tmp_outdir_prefix,
+ arvrunner.runtimeContext.match_local_docker)
except Exception as e:
raise Exception("Docker image %s is not available\n%s" % (img, e) )
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list