[ARVADOS] updated: 1.3.0-2242-g1d9e4de7a
Git user
git at public.arvados.org
Mon Mar 2 18:01:07 UTC 2020
Summary of changes:
doc/api/index.html.textile.liquid | 4 ++++
go.mod | 2 +-
go.sum | 4 ++--
lib/config/export.go | 9 ++++++++-
services/login-sync/Gemfile.lock | 4 ++--
5 files changed, 17 insertions(+), 6 deletions(-)
discards b92483b252dc14a1616a8260d2fe6c929075760a (commit)
discards f992a40fc9d55652558982ecaf8df66a5497dac7 (commit)
via 1d9e4de7a4ff994cfc7a9319dcae56bb26c272b3 (commit)
via 6bc3d81b5d6ce487356a252b9db4ad956a91c453 (commit)
via 7b4082f90082bf9f14ba629ef455cd6e6e82c6a8 (commit)
via 894d7e9c3adfa29f1373a96e54efb45a9a553083 (commit)
via 01d8fcb8fa18e413660164174b0b91593109a271 (commit)
via 1de238b16d94b8ce1e374b8e745671ef46d549c1 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (b92483b252dc14a1616a8260d2fe6c929075760a)
\
N -- N -- N (1d9e4de7a4ff994cfc7a9319dcae56bb26c272b3)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 1d9e4de7a4ff994cfc7a9319dcae56bb26c272b3
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Feb 28 12:07:35 2020 -0500
16169: Monkey patch load_tool.resolve_and_validate_document to fix bug
There is a bug in upstream cwltool where the version updater needs to
replace the document fragments in the loader index with the updated
ones, but actually it only does it for the root document. Normally we
just fix the bug in upstream but that's challenging because current
cwltool dropped support for Python 2.7 and we're still supporting py2
in Arvados 2.0 (although py2 support will most likely be dropped in
Arvados 2.1). Making a bugfix fork comes with its own
complications (it would need to be added to PyPi) so monkey patching
is the least disruptive fix (and is relatively safe because our
cwltool dependency is pinned to a specific version). This
should be removed as soon as a bugfix goes into upstream cwltool and
we upgrade to it.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 3dd04040a..2b2acd568 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -16,6 +16,43 @@ import sys
import re
import pkg_resources # part of setuptools
+### begin monkey patch ###
+# Monkey patch solution for bug #16169
+#
+# There is a bug in upstream cwltool where the version updater needs
+# to replace the document fragments in the loader index with the
+# updated ones, but actually it only does it for the root document.
+# Normally we just fix the bug in upstream but that's challenging
+# because current cwltool dropped support for Python 2.7 and we're
+# still supporting py2 in Arvados 2.0 (although py2 support will most
+# likely be dropped in Arvados 2.1). Making a bugfix fork comes with
+# its own complications (it would need to be added to PyPi) so monkey
+# patching is the least disruptive fix (and is relatively safe because
+# our cwltool dependency is pinned to a specific version). This
+# should be removed as soon as a bugfix goes into upstream cwltool and
+# we upgrade to it.
+#
+import cwltool.load_tool
+from cwltool.utils import visit_class
+from six.moves import urllib
+original_resolve_and_validate_document = cwltool.load_tool.resolve_and_validate_document
+def wrapped_resolve_and_validate_document(
+ loadingContext, # type: LoadingContext
+ workflowobj, # type: Union[CommentedMap, CommentedSeq]
+ uri, # type: Text
+ preprocess_only=False, # type: bool
+ skip_schemas=None, # type: Optional[bool]
+ ):
+ loadingContext, uri = original_resolve_and_validate_document(loadingContext, workflowobj, uri, preprocess_only, skip_schemas)
+ if loadingContext.do_update in (True, None):
+ fileuri = urllib.parse.urldefrag(uri)[0]
+ def update_index(pr):
+ loadingContext.loader.idx[pr["id"]] = pr
+ visit_class(loadingContext.loader.idx[fileuri], ("CommandLineTool", "Workflow", "ExpressionTool"), update_index)
+ return loadingContext, uri
+cwltool.load_tool.resolve_and_validate_document = wrapped_resolve_and_validate_document
+### end monkey patch ###
+
from schema_salad.sourceline import SourceLine
import schema_salad.validate as validate
import cwltool.main
diff --git a/sdk/cwl/tests/wf/16169-step.cwl b/sdk/cwl/tests/wf/16169-step.cwl
index 8386df517..ce6f2c0c9 100644
--- a/sdk/cwl/tests/wf/16169-step.cwl
+++ b/sdk/cwl/tests/wf/16169-step.cwl
@@ -6,10 +6,12 @@ class: CommandLineTool
cwlVersion: v1.0
requirements:
InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: debian:stretch-slim
inputs:
d: Directory
outputs:
out: stdout
stdout: output.txt
arguments:
- [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]
\ No newline at end of file
+ [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index c89fa644c..6673888ab 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -390,7 +390,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
- logger.error(error.message)
+ logger.error(str(error))
sys.exit(1)
if not docker_image_compatible(api, image_hash):
commit 6bc3d81b5d6ce487356a252b9db4ad956a91c453
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Wed Feb 26 20:58:59 2020 -0300
16169: Adds test exposing the requirement propagation bug.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/sdk/cwl/tests/16169-no-listing-hint.cwl b/sdk/cwl/tests/16169-no-listing-hint.cwl
new file mode 100644
index 000000000..fe4f991ff
--- /dev/null
+++ b/sdk/cwl/tests/16169-no-listing-hint.cwl
@@ -0,0 +1,24 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+inputs:
+ d: Directory
+steps:
+ step1:
+ in:
+ d: d
+ out: [out]
+ run: wf/16169-step.cwl
+outputs:
+ out:
+ type: File
+ outputSource: step1/out
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 99aee3795..df9fac842 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -310,3 +310,15 @@
should_fail: true
tool: 15295-bad-keep-ref.cwl
doc: Test checking for invalid keepref
+
+- job: listing-job.yml
+ output: {
+ "out": {
+ "class": "File",
+ "location": "output.txt",
+ "size": 5,
+ "checksum": "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1"
+ }
+ }
+ tool: 16169-no-listing-hint.cwl
+ doc: "Test cwltool:LoadListingRequirement propagation"
diff --git a/sdk/cwl/tests/wf/16169-step.cwl b/sdk/cwl/tests/wf/16169-step.cwl
new file mode 100644
index 000000000..8386df517
--- /dev/null
+++ b/sdk/cwl/tests/wf/16169-step.cwl
@@ -0,0 +1,15 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+ InlineJavascriptRequirement: {}
+inputs:
+ d: Directory
+outputs:
+ out: stdout
+stdout: output.txt
+arguments:
+ [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]
\ No newline at end of file
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list