[ARVADOS] updated: 1.3.0-233-gd1993ffdc
Git user
git at public.curoverse.com
Mon Feb 4 14:23:28 EST 2019
Summary of changes:
sdk/cwl/arvados_cwl/runner.py | 2 +-
sdk/cwl/test_with_arvbox.sh | 39 +++++++++++++++++++++++++++----
sdk/dev-jobs.dockerfile | 11 +++++++--
sdk/python/arvados/commands/keepdocker.py | 2 +-
4 files changed, 45 insertions(+), 9 deletions(-)
via d1993ffdca3bc9036b1915bae6360dee145ee9f2 (commit)
via caeb94881de75f2f8ee0156951c7f54006499e15 (commit)
from 1e8da3c17b221eba0288ec3b6fd5769df2598531 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d1993ffdca3bc9036b1915bae6360dee145ee9f2
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Feb 4 14:22:58 2019 -0500
13306: select between "conformance" and "integration" in test_with_arvbox.sh
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index aa98252c0..936560366 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -14,6 +14,7 @@ leave_running=0
config=dev
tag="latest"
pythoncmd=python
+suite=conformance
while test -n "$1" ; do
arg="$1"
@@ -42,8 +43,12 @@ while test -n "$1" ; do
pythoncmd=$2
shift ; shift
;;
+ --suite)
+ suite=$2
+ shift ; shift
+ ;;
-h|--help)
- echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python[23]]"
+ echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python(2|3)] [--suite (integration|conformance)]"
exit
;;
*)
@@ -77,6 +82,8 @@ if test $config = dev ; then
pip_install \$(ls -r dist/arvados-cwl-runner-*.tar.gz | head -n1)
fi
+set -x
+
if [ \$PYCMD = "python3" ]; then
pip3 install cwltest
else
@@ -119,7 +126,12 @@ EOF2
chmod +x /tmp/cwltest/arv-cwl-containers
env
-exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-containers EXTRA=--compute-checksum $@
+if [[ "$suite" = "conformance" ]] ; then
+ exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-containers EXTRA=--compute-checksum $@
+elif [[ "$suite" = "integration" ]] ; then
+ cd /usr/src/arvados/sdk/cwl/tests
+ exec ./arvados-tests.sh $@
+fi
EOF
CODE=$?
commit caeb94881de75f2f8ee0156951c7f54006499e15
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Feb 4 13:27:09 2019 -0500
13306: test_with_arvbox.sh can test either python2 or python3
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index b65e2c58e..ad8e903ba 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -467,7 +467,7 @@ class Runner(Process):
if "cwl.output.json" in outc:
with outc.open("cwl.output.json", "rb") as f:
if f.size() > 0:
- outputs = json.load(f)
+ outputs = json.loads(f.read().decode())
def keepify(fileobj):
path = fileobj["location"]
if not path.startswith("keep:"):
diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index a6f4022cc..aa98252c0 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -13,6 +13,7 @@ reset_container=1
leave_running=0
config=dev
tag="latest"
+pythoncmd=python
while test -n "$1" ; do
arg="$1"
@@ -33,8 +34,16 @@ while test -n "$1" ; do
tag=$2
shift ; shift
;;
+ --build)
+ build=1
+ shift
+ ;;
+ --pythoncmd)
+ pythoncmd=$2
+ shift ; shift
+ ;;
-h|--help)
- echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag]"
+ echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python[23]]"
exit
;;
*)
@@ -60,13 +69,19 @@ set -eu -o pipefail
. /usr/local/lib/arvbox/common.sh
+export PYCMD=$pythoncmd
+
if test $config = dev ; then
cd /usr/src/arvados/sdk/cwl
- python setup.py sdist
+ \$PYCMD setup.py sdist
pip_install \$(ls -r dist/arvados-cwl-runner-*.tar.gz | head -n1)
fi
-pip install cwltest
+if [ \$PYCMD = "python3" ]; then
+ pip3 install cwltest
+else
+ pip3 install cwltest
+fi
mkdir -p /tmp/cwltest
cd /tmp/cwltest
@@ -80,7 +95,9 @@ export ARVADOS_API_HOST_INSECURE=1
export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados/superuser_token)
-if test "$tag" = "latest" ; then
+if test -n "$build" ; then
+ /usr/src/arvados/build/build-dev-docker-jobs-image.sh
+elif test "$tag" = "latest" ; then
arv-keepdocker --pull arvados/jobs $tag
else
jobsimg=\$(curl https://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])")
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index 9c4a5aa33..f9e370794 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -20,8 +20,15 @@ ENV DEBIAN_FRONTEND noninteractive
ARG pythoncmd=python
-RUN apt-get update -q && apt-get install -qy git ${pythoncmd}-pip python-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
-RUN if [ "$pythoncmd" = "python3" ]; then pip3 install -U setuptools six requests; else pip install -U setuptools six requests; fi
+RUN apt-get update -q && apt-get install -qy --no-install-recommends \
+ git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \
+ libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential
+
+RUN if [ "$pythoncmd" = "python3" ]; then \
+ pip3 install -U setuptools six requests ; \
+ else \
+ pip install -U setuptools six requests ; \
+ fi
ARG sdk
ARG runner
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 2d306d516..811f096c6 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -504,7 +504,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
else:
json_filename = raw_image_hash + '/json'
json_file = image_tar.extractfile(image_tar.getmember(json_filename))
- image_metadata = json.load(json_file)
+ image_metadata = json.loads(json_file.read().decode())
json_file.close()
image_tar.close()
link_base = {'head_uuid': coll_uuid, 'properties': {}}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list