[ARVADOS] updated: 1.3.0-233-gd1993ffdc

Git user git at public.curoverse.com
Mon Feb 4 14:23:28 EST 2019


Summary of changes:
 sdk/cwl/arvados_cwl/runner.py             |  2 +-
 sdk/cwl/test_with_arvbox.sh               | 39 +++++++++++++++++++++++++++----
 sdk/dev-jobs.dockerfile                   | 11 +++++++--
 sdk/python/arvados/commands/keepdocker.py |  2 +-
 4 files changed, 45 insertions(+), 9 deletions(-)

       via  d1993ffdca3bc9036b1915bae6360dee145ee9f2 (commit)
       via  caeb94881de75f2f8ee0156951c7f54006499e15 (commit)
      from  1e8da3c17b221eba0288ec3b6fd5769df2598531 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit d1993ffdca3bc9036b1915bae6360dee145ee9f2
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Mon Feb 4 14:22:58 2019 -0500

    13306: select between "conformance" and "integration" in test_with_arvbox.sh
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index aa98252c0..936560366 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -14,6 +14,7 @@ leave_running=0
 config=dev
 tag="latest"
 pythoncmd=python
+suite=conformance
 
 while test -n "$1" ; do
     arg="$1"
@@ -42,8 +43,12 @@ while test -n "$1" ; do
             pythoncmd=$2
             shift ; shift
             ;;
+        --suite)
+            suite=$2
+            shift ; shift
+            ;;
         -h|--help)
-            echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python[23]]"
+            echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python(2|3)] [--suite (integration|conformance)]"
             exit
             ;;
         *)
@@ -77,6 +82,8 @@ if test $config = dev ; then
   pip_install \$(ls -r dist/arvados-cwl-runner-*.tar.gz | head -n1)
 fi
 
+set -x
+
 if [ \$PYCMD = "python3" ]; then
     pip3 install cwltest
 else
@@ -119,7 +126,12 @@ EOF2
 chmod +x /tmp/cwltest/arv-cwl-containers
 
 env
-exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-containers EXTRA=--compute-checksum $@
+if [[ "$suite" = "conformance" ]] ; then
+   exec ./run_test.sh RUNNER=/tmp/cwltest/arv-cwl-containers EXTRA=--compute-checksum $@
+elif [[ "$suite" = "integration" ]] ; then
+   cd /usr/src/arvados/sdk/cwl/tests
+   exec ./arvados-tests.sh $@
+fi
 EOF
 
 CODE=$?

commit caeb94881de75f2f8ee0156951c7f54006499e15
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Mon Feb 4 13:27:09 2019 -0500

    13306: test_with_arvbox.sh can test either python2 or python3
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index b65e2c58e..ad8e903ba 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -467,7 +467,7 @@ class Runner(Process):
             if "cwl.output.json" in outc:
                 with outc.open("cwl.output.json", "rb") as f:
                     if f.size() > 0:
-                        outputs = json.load(f)
+                        outputs = json.loads(f.read().decode())
             def keepify(fileobj):
                 path = fileobj["location"]
                 if not path.startswith("keep:"):
diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index a6f4022cc..aa98252c0 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -13,6 +13,7 @@ reset_container=1
 leave_running=0
 config=dev
 tag="latest"
+pythoncmd=python
 
 while test -n "$1" ; do
     arg="$1"
@@ -33,8 +34,16 @@ while test -n "$1" ; do
             tag=$2
             shift ; shift
             ;;
+        --build)
+            build=1
+            shift
+            ;;
+        --pythoncmd)
+            pythoncmd=$2
+            shift ; shift
+            ;;
         -h|--help)
-            echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag]"
+            echo "$0 [--no-reset-container] [--leave-running] [--config dev|localdemo] [--tag docker_tag] [--build] [--pythoncmd python[23]]"
             exit
             ;;
         *)
@@ -60,13 +69,19 @@ set -eu -o pipefail
 
 . /usr/local/lib/arvbox/common.sh
 
+export PYCMD=$pythoncmd
+
 if test $config = dev ; then
   cd /usr/src/arvados/sdk/cwl
-  python setup.py sdist
+  \$PYCMD setup.py sdist
   pip_install \$(ls -r dist/arvados-cwl-runner-*.tar.gz | head -n1)
 fi
 
-pip install cwltest
+if [ \$PYCMD = "python3" ]; then
+    pip3 install cwltest
+else
+    pip3 install cwltest
+fi
 
 mkdir -p /tmp/cwltest
 cd /tmp/cwltest
@@ -80,7 +95,9 @@ export ARVADOS_API_HOST_INSECURE=1
 export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados/superuser_token)
 
 
-if test "$tag" = "latest" ; then
+if test -n "$build" ; then
+   /usr/src/arvados/build/build-dev-docker-jobs-image.sh
+elif test "$tag" = "latest" ; then
   arv-keepdocker --pull arvados/jobs $tag
 else
   jobsimg=\$(curl https://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])")
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index 9c4a5aa33..f9e370794 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -20,8 +20,15 @@ ENV DEBIAN_FRONTEND noninteractive
 
 ARG pythoncmd=python
 
-RUN apt-get update -q && apt-get install -qy git ${pythoncmd}-pip python-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
-RUN if [ "$pythoncmd" = "python3" ]; then pip3 install -U setuptools six requests; else pip install -U setuptools six requests; fi
+RUN apt-get update -q && apt-get install -qy --no-install-recommends \
+    git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \
+    libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential
+
+RUN if [ "$pythoncmd" = "python3" ]; then \
+       pip3 install -U setuptools six requests ; \
+    else \
+       pip install -U setuptools six requests ; \
+    fi
 
 ARG sdk
 ARG runner
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 2d306d516..811f096c6 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -504,7 +504,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
         else:
             json_filename = raw_image_hash + '/json'
         json_file = image_tar.extractfile(image_tar.getmember(json_filename))
-        image_metadata = json.load(json_file)
+        image_metadata = json.loads(json_file.read().decode())
         json_file.close()
         image_tar.close()
         link_base = {'head_uuid': coll_uuid, 'properties': {}}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list