[ARVADOS] updated: 1.3.0-2238-gb92483b25
Git user
git at public.arvados.org
Mon Mar 2 16:11:47 UTC 2020
Summary of changes:
sdk/cwl/arvados_cwl/__init__.py | 37 +++++++++++++++++++++++++++++++
sdk/cwl/tests/wf/16169-step.cwl | 4 +++-
sdk/python/arvados/commands/keepdocker.py | 2 +-
services/login-sync/Gemfile.lock | 4 ++--
4 files changed, 43 insertions(+), 4 deletions(-)
discards b12b6c014f0e26fb4c2c2a5ad27a36c3685babf1 (commit)
discards 0ec0f9bf97f651721dd9cca92ee088da3a122463 (commit)
via b92483b252dc14a1616a8260d2fe6c929075760a (commit)
via f992a40fc9d55652558982ecaf8df66a5497dac7 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (b12b6c014f0e26fb4c2c2a5ad27a36c3685babf1)
\
N -- N -- N (b92483b252dc14a1616a8260d2fe6c929075760a)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b92483b252dc14a1616a8260d2fe6c929075760a
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Feb 28 12:07:35 2020 -0500
16169: Monkey patch load_tool.resolve_and_validate_document to fix bug
There is a bug in upstream cwltool where the version updater needs to
replace the document fragments in the loader index with the updated
ones, but actually it only does it for the root document. Normally we
just fix the bug in upstream but that's challenging because current
cwltool dropped support for Python 2.7 and we're still supporting py2
in Arvados 2.0 (although py2 support will most likely be dropped in
Arvados 2.1). Making a bugfix fork comes with its own
complications (it would need to be added to PyPi) so monkey patching
is the least disruptive fix (and is relatively safe because our
cwltool dependency is pinned to a specific version). This
should be removed as soon as a bugfix goes into upstream cwltool and
we upgrade to it.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 3dd04040a..2b2acd568 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -16,6 +16,43 @@ import sys
import re
import pkg_resources # part of setuptools
+### begin monkey patch ###
+# Monkey patch solution for bug #16169
+#
+# There is a bug in upstream cwltool where the version updater needs
+# to replace the document fragments in the loader index with the
+# updated ones, but actually it only does it for the root document.
+# Normally we just fix the bug in upstream but that's challenging
+# because current cwltool dropped support for Python 2.7 and we're
+# still supporting py2 in Arvados 2.0 (although py2 support will most
+# likely be dropped in Arvados 2.1). Making a bugfix fork comes with
+# its own complications (it would need to be added to PyPi) so monkey
+# patching is the least disruptive fix (and is relatively safe because
+# our cwltool dependency is pinned to a specific version). This
+# should be removed as soon as a bugfix goes into upstream cwltool and
+# we upgrade to it.
+#
+import cwltool.load_tool
+from cwltool.utils import visit_class
+from six.moves import urllib
+original_resolve_and_validate_document = cwltool.load_tool.resolve_and_validate_document
+def wrapped_resolve_and_validate_document(
+ loadingContext, # type: LoadingContext
+ workflowobj, # type: Union[CommentedMap, CommentedSeq]
+ uri, # type: Text
+ preprocess_only=False, # type: bool
+ skip_schemas=None, # type: Optional[bool]
+ ):
+ loadingContext, uri = original_resolve_and_validate_document(loadingContext, workflowobj, uri, preprocess_only, skip_schemas)
+ if loadingContext.do_update in (True, None):
+ fileuri = urllib.parse.urldefrag(uri)[0]
+ def update_index(pr):
+ loadingContext.loader.idx[pr["id"]] = pr
+ visit_class(loadingContext.loader.idx[fileuri], ("CommandLineTool", "Workflow", "ExpressionTool"), update_index)
+ return loadingContext, uri
+cwltool.load_tool.resolve_and_validate_document = wrapped_resolve_and_validate_document
+### end monkey patch ###
+
from schema_salad.sourceline import SourceLine
import schema_salad.validate as validate
import cwltool.main
diff --git a/sdk/cwl/tests/wf/16169-step.cwl b/sdk/cwl/tests/wf/16169-step.cwl
index 8386df517..ce6f2c0c9 100644
--- a/sdk/cwl/tests/wf/16169-step.cwl
+++ b/sdk/cwl/tests/wf/16169-step.cwl
@@ -6,10 +6,12 @@ class: CommandLineTool
cwlVersion: v1.0
requirements:
InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: debian:stretch-slim
inputs:
d: Directory
outputs:
out: stdout
stdout: output.txt
arguments:
- [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]
\ No newline at end of file
+ [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index c89fa644c..6673888ab 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -390,7 +390,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
- logger.error(error.message)
+ logger.error(str(error))
sys.exit(1)
if not docker_image_compatible(api, image_hash):
diff --git a/services/login-sync/Gemfile.lock b/services/login-sync/Gemfile.lock
index 28cdff6fd..c1b63de3c 100644
--- a/services/login-sync/Gemfile.lock
+++ b/services/login-sync/Gemfile.lock
@@ -1,7 +1,7 @@
PATH
remote: .
specs:
- arvados-login-sync (1.5.0.dev20200114213539)
+ arvados-login-sync (1.5.0.dev20200118023341)
arvados (~> 1.3.0, >= 1.3.0)
faraday (< 0.16)
signet (< 0.12)
@@ -86,4 +86,4 @@ DEPENDENCIES
rake
BUNDLED WITH
- 1.11
+ 1.13.6
commit f992a40fc9d55652558982ecaf8df66a5497dac7
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Wed Feb 26 20:58:59 2020 -0300
16169: Adds test exposing the requirement propagation bug.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/sdk/cwl/tests/16169-no-listing-hint.cwl b/sdk/cwl/tests/16169-no-listing-hint.cwl
new file mode 100644
index 000000000..fe4f991ff
--- /dev/null
+++ b/sdk/cwl/tests/16169-no-listing-hint.cwl
@@ -0,0 +1,24 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+inputs:
+ d: Directory
+steps:
+ step1:
+ in:
+ d: d
+ out: [out]
+ run: wf/16169-step.cwl
+outputs:
+ out:
+ type: File
+ outputSource: step1/out
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index 99aee3795..df9fac842 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -310,3 +310,15 @@
should_fail: true
tool: 15295-bad-keep-ref.cwl
doc: Test checking for invalid keepref
+
+- job: listing-job.yml
+ output: {
+ "out": {
+ "class": "File",
+ "location": "output.txt",
+ "size": 5,
+ "checksum": "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1"
+ }
+ }
+ tool: 16169-no-listing-hint.cwl
+ doc: "Test cwltool:LoadListingRequirement propagation"
diff --git a/sdk/cwl/tests/wf/16169-step.cwl b/sdk/cwl/tests/wf/16169-step.cwl
new file mode 100644
index 000000000..8386df517
--- /dev/null
+++ b/sdk/cwl/tests/wf/16169-step.cwl
@@ -0,0 +1,15 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+ InlineJavascriptRequirement: {}
+inputs:
+ d: Directory
+outputs:
+ out: stdout
+stdout: output.txt
+arguments:
+ [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]
\ No newline at end of file
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list