[ARVADOS] updated: 1.3.0-170-gdb09e5f78
Git user
git at public.curoverse.com
Fri Jan 25 13:03:31 EST 2019
Summary of changes:
build/build-dev-docker-jobs-image.sh | 8 +++++++-
sdk/cwl/arvados_cwl/__init__.py | 24 ++++++++++++------------
sdk/cwl/arvados_cwl/arvcontainer.py | 2 +-
sdk/cwl/arvados_cwl/fsaccess.py | 10 +++++-----
sdk/cwl/arvados_cwl/pathmapper.py | 2 +-
sdk/dev-jobs.dockerfile | 14 ++++++++------
sdk/python/arvados/commands/keepdocker.py | 1 +
7 files changed, 35 insertions(+), 26 deletions(-)
via db09e5f787a1b2d0e02abb627bf25054a047f7db (commit)
via fa5e2a328b26c71194bf676d850b3971542123fc (commit)
via 88cc6dd9d342aa997bafb5201dc2b9a7609805d5 (commit)
from 8c593d84f88aa5cd87de0acedffdf867deca51f7 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit db09e5f787a1b2d0e02abb627bf25054a047f7db
Author: Eric Biagiotti <ebiagiotti at veritasgenetcs.com>
Date: Fri Jan 25 13:03:12 2019 -0500
13306: Adding support for building the dev docker jobs image with py3
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index 9393c1acc..2e4c45738 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -16,6 +16,7 @@ Syntax:
WORKSPACE=path Path to the Arvados source tree to build packages from
CWLTOOL=path (optional) Path to cwltool git repository.
SALAD=path (optional) Path to schema_salad git repository.
+PYCMD=pythonexec (optional) Specify the python executable to use in the docker image. Defaults to "python".
EOF
@@ -35,6 +36,11 @@ fi
cd "$WORKSPACE"
+py=python
+if [[ -n "$PYCMD" ]] ; then
+ py="$PYCMD" ;
+fi
+
(cd sdk/python && python setup.py sdist)
sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
@@ -69,6 +75,6 @@ if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
fi
-docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
+docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
echo arv-keepdocker arvados/jobs $cwl_runner_version
arv-keepdocker arvados/jobs $cwl_runner_version
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index d33956ccc..ba1cbfec1 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -18,9 +18,11 @@ MAINTAINER Ward Vandewege <ward at curoverse.com>
ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
+ARG pythoncmd=python
-RUN pip install -U setuptools six requests
+RUN echo $pythoncmd
+RUN apt-get update -q && apt-get install -qy git ${pythoncmd}-pip python-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
+RUN if [ "$pythoncmd" = "python3" ]; then pip3 install -U setuptools six requests; else pip install -U setuptools six requests; fi
ARG sdk
ARG runner
@@ -32,10 +34,10 @@ ADD cwl/salad_dist/$salad /tmp/
ADD cwl/cwltool_dist/$cwltool /tmp/
ADD cwl/dist/$runner /tmp/
-RUN cd /tmp/arvados-python-client-* && python setup.py install
-RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && python setup.py install ; fi
-RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && python setup.py install ; fi
-RUN cd /tmp/arvados-cwl-runner-* && python setup.py install
+RUN cd /tmp/arvados-python-client-* && $pythoncmd setup.py install
+RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pythoncmd setup.py install ; fi
+RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pythoncmd setup.py install ; fi
+RUN cd /tmp/arvados-cwl-runner-* && $pythoncmd setup.py install
# Install dependencies and set up system.
RUN /usr/sbin/adduser --disabled-password \
commit fa5e2a328b26c71194bf676d850b3971542123fc
Author: Eric Biagiotti <ebiagiotti at veritasgenetcs.com>
Date: Thu Jan 24 09:38:25 2019 -0500
13306: Unicode fix for docker image generation in py3 environment
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 55fc6b626..2d306d516 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -133,6 +133,7 @@ def docker_images():
next(list_output) # Ignore the header line
for line in list_output:
words = line.split()
+ words = [word.decode() for word in words]
size_index = len(words) - 2
repo, tag, imageid = words[:3]
ctime = ' '.join(words[3:size_index])
commit 88cc6dd9d342aa997bafb5201dc2b9a7609805d5
Author: Eric Biagiotti <ebiagiotti at veritasgenetcs.com>
Date: Thu Jan 24 09:37:13 2019 -0500
13306: Unicode fixes for the conformance tests
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 09deabd22..52fd4d21a 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -69,9 +69,9 @@ def versionstring():
def arg_parser(): # type: () -> argparse.ArgumentParser
parser = argparse.ArgumentParser(description='Arvados executor for Common Workflow Language')
- parser.add_argument("--basedir", type=str,
+ parser.add_argument("--basedir",
help="Base directory used to resolve relative references in the input, default to directory of input object file or current directory (if inputs piped/provided on command line).")
- parser.add_argument("--outdir", type=str, default=os.path.abspath('.'),
+ parser.add_argument("--outdir", default=os.path.abspath('.'),
help="Output directory, default current directory")
parser.add_argument("--eval-timeout",
@@ -102,9 +102,9 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
default=True, dest="enable_reuse",
help="Disable job or container reuse")
- parser.add_argument("--project-uuid", type=str, metavar="UUID", help="Project that will own the workflow jobs, if not provided, will go to home project.")
- parser.add_argument("--output-name", type=str, help="Name to use for collection that stores the final output.", default=None)
- parser.add_argument("--output-tags", type=str, help="Tags for the final output collection separated by commas, e.g., '--output-tags tag0,tag1,tag2'.", default=None)
+ parser.add_argument("--project-uuid", metavar="UUID", help="Project that will own the workflow jobs, if not provided, will go to home project.")
+ parser.add_argument("--output-name", help="Name to use for collection that stores the final output.", default=None)
+ parser.add_argument("--output-tags", help="Tags for the final output collection separated by commas, e.g., '--output-tags tag0,tag1,tag2'.", default=None)
parser.add_argument("--ignore-docker-for-reuse", action="store_true",
help="Ignore Docker image version when deciding whether to reuse past jobs.",
default=False)
@@ -131,7 +131,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
exgroup.add_argument("--no-log-timestamps", action="store_false", help="No timestamp on logging lines",
default=True, dest="log_timestamps")
- parser.add_argument("--api", type=str,
+ parser.add_argument("--api",
default=None, dest="work_api",
choices=("jobs", "containers"),
help="Select work submission API. Default is 'jobs' if that API is available, otherwise 'containers'.")
@@ -144,7 +144,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help="RAM (in MiB) required for the workflow runner job (default 1024)",
default=None)
- parser.add_argument("--submit-runner-image", type=str,
+ parser.add_argument("--submit-runner-image",
help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
default=None)
@@ -153,11 +153,11 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
default=False)
exgroup = parser.add_mutually_exclusive_group()
- exgroup.add_argument("--submit-request-uuid", type=str,
+ exgroup.add_argument("--submit-request-uuid",
default=None,
help="Update and commit to supplied container request instead of creating a new one (containers API only).",
metavar="UUID")
- exgroup.add_argument("--submit-runner-cluster", type=str,
+ exgroup.add_argument("--submit-runner-cluster",
help="Submit workflow runner to a remote cluster (containers API only)",
default=None,
metavar="CLUSTER_ID")
@@ -166,7 +166,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
default=None,
help="Collection cache size (in MiB, default 256).")
- parser.add_argument("--name", type=str,
+ parser.add_argument("--name",
help="Name to use for workflow execution instance.",
default=None)
@@ -178,7 +178,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
parser.add_argument("--enable-dev", action="store_true",
help="Enable loading and running development versions "
"of CWL spec.", default=False)
- parser.add_argument('--storage-classes', default="default", type=str,
+ parser.add_argument('--storage-classes', default="default",
help="Specify comma separated list of storage classes to be used when saving workflow output to Keep.")
parser.add_argument("--intermediate-output-ttl", type=int, metavar="N",
@@ -211,7 +211,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
default=False, dest="trash_intermediate",
help="Do not trash intermediate outputs (default).")
- parser.add_argument("workflow", type=str, default=None, help="The workflow to execute")
+ parser.add_argument("workflow", default=None, help="The workflow to execute")
parser.add_argument("job_order", nargs=argparse.REMAINDER, help="The input object to the workflow.")
return parser
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 47fbfa5a7..a45321d37 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -162,7 +162,7 @@ class ArvadosContainer(JobBase):
}
else:
with vwd.open(p.target, "w") as n:
- n.write(p.resolved.encode("utf-8"))
+ n.write(p.resolved)
def keepemptydirs(p):
if isinstance(p, arvados.collection.RichCollectionBase):
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index dbcc66245..fc0bbb80b 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -288,21 +288,21 @@ pipeline_template_uuid_pattern = re.compile(r'[a-z0-9]{5}-p5p6p-[a-z0-9]{15}')
def collectionResolver(api_client, document_loader, uri, num_retries=4):
if uri.startswith("keep:") or uri.startswith("arvwf:"):
- return uri
+ return uri.encode("utf-8").decode()
if workflow_uuid_pattern.match(uri):
- return "arvwf:%s#main" % (uri)
+ return u"arvwf:%s#main" % (uri)
if pipeline_template_uuid_pattern.match(uri):
pt = api_client.pipeline_templates().get(uuid=uri).execute(num_retries=num_retries)
- return "keep:" + viewvalues(pt["components"])[0]["script_parameters"]["cwl:tool"]
+ return u"keep:" + viewvalues(pt["components"])[0]["script_parameters"]["cwl:tool"]
p = uri.split("/")
if arvados.util.keep_locator_pattern.match(p[0]):
- return "keep:%s" % (uri)
+ return u"keep:%s" % (uri)
if arvados.util.collection_uuid_pattern.match(p[0]):
- return "keep:%s%s" % (api_client.collections().
+ return u"keep:%s%s" % (api_client.collections().
get(uuid=p[0]).execute()["portable_data_hash"],
uri[len(p[0]):])
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 2be850c5c..e0445febd 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -120,7 +120,7 @@ class ArvPathMapper(PathMapper):
remap.append((obj["location"], path + "/" + obj["basename"]))
elif obj["location"].startswith("_:") and "contents" in obj:
with c.open(path + "/" + obj["basename"], "w") as f:
- f.write(obj["contents"].encode("utf-8"))
+ f.write(obj["contents"])
remap.append((obj["location"], path + "/" + obj["basename"]))
else:
raise SourceLine(obj, "location", WorkflowException).makeError("Don't know what to do with '%s'" % obj["location"])
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list