[ARVADOS] updated: 1.3.0-2257-gf3b02f85c
Git user
git at public.arvados.org
Mon Mar 9 22:07:33 UTC 2020
Summary of changes:
.../app/controllers/application_controller.rb | 1 +
.../app/controllers/collections_controller.rb | 2 +-
.../app/controllers/virtual_machines_controller.rb | 4 +-
.../application/{error.html.erb => error.text.erb} | 8 +-
.../app/views/users/_virtual_machines.html.erb | 2 +-
.../controllers/collections_controller_test.rb | 4 +-
build/run-tests.sh | 4 +-
doc/api/index.html.textile.liquid | 4 +
go.mod | 2 +-
go.sum | 4 +-
lib/config/config.default.yml | 21 +++++
lib/config/export.go | 10 ++-
lib/config/generated_config.go | 21 +++++
lib/controller/handler_test.go | 21 +++++
lib/dispatchcloud/container/queue.go | 7 +-
lib/dispatchcloud/container/queue_test.go | 7 ++
sdk/cli/arvados-cli.gemspec | 7 +-
sdk/cwl/arvados_cwl/__init__.py | 37 +++++++++
.../sub.cwl => 16169-no-listing-hint.cwl} | 19 +++--
sdk/cwl/tests/arvados-tests.yml | 12 +++
sdk/cwl/tests/test_container.py | 9 ++-
sdk/cwl/tests/test_submit.py | 3 +-
.../tests/wf/{listing_none.cwl => 16169-step.cwl} | 8 +-
sdk/cwl/tests/wf/scatter2_subwf.cwl | 4 +-
sdk/go/arvados/api.go | 1 +
sdk/go/arvados/config.go | 1 +
sdk/go/arvados/container.go | 6 +-
sdk/python/arvados/commands/keepdocker.py | 2 +-
sdk/python/tests/run_test_server.py | 2 +-
sdk/ruby/arvados.gemspec | 6 +-
.../app/controllers/user_sessions_controller.rb | 3 +-
services/api/app/models/user.rb | 8 +-
services/keepproxy/keepproxy.go | 2 +-
services/login-sync/.gitignore | 1 +
services/login-sync/Gemfile.lock | 89 ----------------------
services/login-sync/arvados-login-sync.gemspec | 7 +-
services/nodemanager/arvados_version.py | 17 ++++-
tools/arvbox/lib/arvbox/docker/common.sh | 4 +-
tools/arvbox/lib/arvbox/docker/service/nginx/run | 7 +-
.../lib/arvbox/docker/service/sdk/run-service | 6 +-
tools/crunchstat-summary/arvados_version.py | 17 ++++-
41 files changed, 252 insertions(+), 148 deletions(-)
copy apps/workbench/app/views/application/{error.html.erb => error.text.erb} (59%)
copy sdk/cwl/tests/{secondary/sub.cwl => 16169-no-listing-hint.cwl} (58%)
copy sdk/cwl/tests/wf/{listing_none.cwl => 16169-step.cwl} (70%)
delete mode 100644 services/login-sync/Gemfile.lock
discards 94c1475c0f16c2b79bde2783198c201c3570e1fb (commit)
discards b4595c075c9883206e4c48cfc45de8cbe0c6f7e8 (commit)
discards b6e38668ce1190efe275283982bd83030dcf800d (commit)
discards 63b1854120110f1273e6a54458388e055717bf41 (commit)
via f3b02f85ce244e22033e40e59caf87228b11faee (commit)
via a3c1bd10eaad38b1d05b2c7a1c76619776651059 (commit)
via 28528aed21ad771e144d5d01fb8691c8cafba376 (commit)
via c82d1b4384ee253576f4693d76eef389d15e0407 (commit)
via 0bb92232b7fbfc5ccf5fcbe131d89ce15a88cf02 (commit)
via 971223b2b9e05fb7e1839f41e58d3060623698bb (commit)
via 26c62aa2c173b981f335580885f732a81611e89b (commit)
via 12eec0fb0fc7cce9d012c81b44a9684ea45d6926 (commit)
via 9c9eecd17f04b21fb63e331b3fabd617f9157b53 (commit)
via 643b160f9af9a245607d4cbee488d08f8516df5e (commit)
via 44bd1f56f64b6b587e13844945830717e2630e6b (commit)
via d11eb815d1722e50cf5308dc6e9d99e93d3228b7 (commit)
via 5e55bbfce418bbdbd5069a4b6ba14539815e764c (commit)
via af5e26a900bbfa779ad325491d80f43eaf75de1d (commit)
via c56d04266de4e97b1b861c068309476ca562d72f (commit)
via 1d9e4de7a4ff994cfc7a9319dcae56bb26c272b3 (commit)
via 6bc3d81b5d6ce487356a252b9db4ad956a91c453 (commit)
via 7b4082f90082bf9f14ba629ef455cd6e6e82c6a8 (commit)
via 894d7e9c3adfa29f1373a96e54efb45a9a553083 (commit)
via 01d8fcb8fa18e413660164174b0b91593109a271 (commit)
via 1de238b16d94b8ce1e374b8e745671ef46d549c1 (commit)
via fc60d13593b86ac0b1c00da285130c664b541987 (commit)
via 318a73e6f18476f82b0280f23246192c1564b595 (commit)
via 41a79db76ab767e22333121194c45306c9c7c6a3 (commit)
via e9582c08c1fd1a427e6f1308d344523fd452dc8b (commit)
via f8d2329133018a8266d2f29ff24c06671fd7fde0 (commit)
via 1f7fe86e0038996ae18ea57ce4a470a9648681de (commit)
via f8b7dc97730ded2c22bedd1a8fb4f13d388a570f (commit)
via ee2136319b0e79e8f63330211e96448c06f71f39 (commit)
via e20adcd07cefcf4f35d6b778567fcfb46e197246 (commit)
via ae47f5fc8d181bc26e96218286e8801a20bfe8d7 (commit)
via f26e039288e744510a6478b7b2597d3c494fb2d1 (commit)
via dbc875430e4329862d780f12b804b716a90fc651 (commit)
via e470a947218ecf042ca75c69d4579a052199e570 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (94c1475c0f16c2b79bde2783198c201c3570e1fb)
\
N -- N -- N (f3b02f85ce244e22033e40e59caf87228b11faee)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f3b02f85ce244e22033e40e59caf87228b11faee
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Mar 9 18:07:03 2020 -0400
12409: Fix tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index cb6cfbadb..8677db92a 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -18,6 +18,7 @@ import os
import functools
import cwltool.process
import cwltool.secrets
+from cwltool.update import INTERNAL_VERSION
from schema_salad.ref_resolver import Loader
from schema_salad.sourceline import cmap
@@ -60,7 +61,7 @@ class TestContainer(unittest.TestCase):
cwltool.process._names = set()
def helper(self, runner, enable_reuse=True):
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -69,7 +70,7 @@ class TestContainer(unittest.TestCase):
"basedir": "",
"make_fs_access": make_fs_access,
"loader": Loader({}),
- "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
+ "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"basedir": "",
@@ -402,7 +403,7 @@ class TestContainer(unittest.TestCase):
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
tool = cmap({
"inputs": [],
@@ -863,7 +864,7 @@ class TestWorkflow(unittest.TestCase):
"basedir": "",
"make_fs_access": make_fs_access,
"loader": document_loader,
- "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
+ "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
"construct_tool_object": runner.arv_make_tool})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 397ae1422..562664c69 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -18,6 +18,7 @@ import mock
import sys
import unittest
import cwltool.process
+import re
from io import BytesIO
@@ -1312,7 +1313,7 @@ class TestSubmit(unittest.TestCase):
self.assertEqual(exited, 1)
self.assertRegexpMatches(
- capture_stderr.getvalue(),
+ re.sub(r'[ \n]+', ' ', capture_stderr.getvalue()),
r"Expected collection uuid zzzzz-4zz18-zzzzzzzzzzzzzzz to be 99999999999999999999999999999998\+99 but API server reported 99999999999999999999999999999997\+99")
finally:
cwltool_logger.removeHandler(stderr_logger)
diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl
index 7a07de5bd..035c1e97e 100644
--- a/sdk/cwl/tests/wf/scatter2_subwf.cwl
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@ -6,7 +6,7 @@
"$graph": [
{
"class": "Workflow",
- "cwlVersion": "v1.1",
+ "cwlVersion": "v1.2.0-dev1",
"hints": [],
"id": "#main",
"inputs": [
@@ -83,5 +83,5 @@
]
}
],
- "cwlVersion": "v1.1"
+ "cwlVersion": "v1.2.0-dev1"
}
commit a3c1bd10eaad38b1d05b2c7a1c76619776651059
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Feb 27 18:10:22 2020 -0500
12409: Interactive mode testargs works for py3
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 8556011df..74a701d46 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -1265,7 +1265,8 @@ else
${verb}_${target}
;;
*)
- testargs["$target"]="${opts}"
+ argstarget=${target%:py3}
+ testargs["$argstarget"]="${opts}"
tt="${testfuncargs[${target}]}"
tt="${tt:-$target}"
do_$verb $tt
commit 28528aed21ad771e144d5d01fb8691c8cafba376
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Feb 26 14:18:53 2020 -0500
12409: Only run sdk/cwl tests py3
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index a53efb800..8556011df 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -127,7 +127,7 @@ sdk/go/blockdigest
sdk/go/asyncbuf
sdk/go/stats
sdk/go/crunchrunner
-sdk/cwl
+sdk/cwl:py3
sdk/R
sdk/java-v2
tools/sync-groups
@@ -988,7 +988,6 @@ pythonstuff=(
sdk/pam
sdk/python
sdk/python:py3
- sdk/cwl
sdk/cwl:py3
services/dockercleaner:py3
services/fuse
commit c82d1b4384ee253576f4693d76eef389d15e0407
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Feb 26 13:55:46 2020 -0500
12409: Add --devcwl
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index 3b29d39c5..76aa43d61 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -12,6 +12,7 @@ fi
reset_container=1
leave_running=0
config=dev
+devcwl=0
tag="latest"
pythoncmd=python3
suite=conformance
@@ -40,6 +41,10 @@ while test -n "$1" ; do
build=1
shift
;;
+ --devcwl)
+ devcwl=1
+ shift
+ ;;
--pythoncmd)
pythoncmd=$2
shift ; shift
@@ -146,12 +151,18 @@ exec arvados-cwl-runner --api=containers \\\$@
EOF2
chmod +x /tmp/cwltest/arv-cwl-containers
+EXTRA=--compute-checksum
+
+if [[ $devcwl == 1 ]] ; then
+ EXTRA="\$EXTRA --enable-dev"
+fi
+
env
if [[ "$suite" = "integration" ]] ; then
cd /usr/src/arvados/sdk/cwl/tests
exec ./arvados-tests.sh $@
else
- exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-${runapi} EXTRA=--compute-checksum $@
+ exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-${runapi} EXTRA="\$EXTRA" $@
fi
EOF
commit 0bb92232b7fbfc5ccf5fcbe131d89ce15a88cf02
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Feb 26 13:38:05 2020 -0500
12409: test_with_arvbox work in progress
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index 52df80f58..a3d439be6 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -16,7 +16,7 @@ Syntax:
WORKSPACE=path Path to the Arvados source tree to build packages from
CWLTOOL=path (optional) Path to cwltool git repository.
SALAD=path (optional) Path to schema_salad git repository.
-PYCMD=pythonexec (optional) Specify the python executable to use in the docker image. Defaults to "python".
+PYCMD=pythonexec (optional) Specify the python executable to use in the docker image. Defaults to "python3".
EOF
@@ -36,13 +36,13 @@ fi
cd "$WORKSPACE"
-py=python
+py=python3
pipcmd=pip
if [[ -n "$PYCMD" ]] ; then
py="$PYCMD"
- if [[ $py = python3 ]] ; then
- pipcmd=pip3
- fi
+fi
+if [[ $py = python3 ]] ; then
+ pipcmd=pip3
fi
(cd sdk/python && python setup.py sdist)
@@ -79,6 +79,7 @@ if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
fi
-docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
+set -x
+docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
echo arv-keepdocker arvados/jobs $cwl_runner_version
arv-keepdocker arvados/jobs $cwl_runner_version
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index d4bb6d102..38626ba80 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -39,14 +39,11 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==1.0.20190831161204',
- 'schema-salad==4.5.20190815125611',
- 'typing >= 3.6.4',
- 'ruamel.yaml >=0.15.54, <=0.15.77',
+ 'cwltool==2.0.20200224214940',
+ 'schema-salad==5.0.20200220195218',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
- 'ciso8601 >= 2.0.0',
- 'networkx < 2.3'
+ 'ciso8601 >= 2.0.0'
],
extras_require={
':os.name=="posix" and python_version<"3"': ['subprocess32 >= 3.5.1'],
@@ -55,8 +52,8 @@ setup(name='arvados-cwl-runner',
data_files=[
('share/doc/arvados-cwl-runner', ['LICENSE-2.0.txt', 'README.rst']),
],
+ python_requires=">=3.5, <4",
classifiers=[
- 'Programming Language :: Python :: 2',
'Programming Language :: Python :: 3',
],
test_suite='tests',
diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index 39c834ed6..3b29d39c5 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -13,7 +13,7 @@ reset_container=1
leave_running=0
config=dev
tag="latest"
-pythoncmd=python
+pythoncmd=python3
suite=conformance
runapi=containers
@@ -53,7 +53,7 @@ while test -n "$1" ; do
shift ; shift
;;
-h|--help)
- echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python(2|3)] [--suite (integration|conformance-v1.0|conformance-v1.1)]"
+ echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python(2|3)] [--suite (integration|conformance-v1.0|conformance-*)]"
exit
;;
*)
@@ -107,11 +107,12 @@ if [[ "$suite" = "conformance-v1.0" ]] ; then
git clone https://github.com/common-workflow-language/common-workflow-language.git
fi
cd common-workflow-language
-elif [[ "$suite" = "conformance-v1.1" ]] ; then
- if ! test -d cwl-v1.1 ; then
- git clone https://github.com/common-workflow-language/cwl-v1.1.git
+elif [[ "$suite" =~ conformance-(.*) ]] ; then
+ version=\${BASH_REMATCH[1]}
+ if ! test -d cwl-\${version} ; then
+ git clone https://github.com/common-workflow-language/cwl-\${version}.git
fi
- cd cwl-v1.1
+ cd cwl-\${version}
fi
if [[ "$suite" != "integration" ]] ; then
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list