[ARVADOS] created: 0cdbfaa368e00fe2bbc026f55e1027f203284979
Git user
git at public.curoverse.com
Thu Sep 28 15:24:40 EDT 2017
at 0cdbfaa368e00fe2bbc026f55e1027f203284979 (commit)
commit 0cdbfaa368e00fe2bbc026f55e1027f203284979
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Sep 28 12:52:44 2017 -0400
12278: Need to set debug logging in crunch_script.py
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 80235e4..12d74a0 100644
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -109,6 +109,11 @@ def run():
fs_access=make_fs_access(""),
num_retries=runner.num_retries))
+ if debug:
+ logger.setLevel(logging.DEBUG)
+ logging.getLogger('arvados').setLevel(logging.DEBUG)
+ logging.getLogger("cwltool").setLevel(logging.DEBUG)
+
args = argparse.Namespace()
args.project_uuid = arvados.current_job()["owner_uuid"]
args.enable_reuse = enable_reuse
commit 8a9be3ff4744d82018cce0cddf7f97b7cfe86a40
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Sep 28 12:12:33 2017 -0400
12278: Bump schema-salad and cwltool for SourceLine fix.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 813feed..762c6fd 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -51,8 +51,8 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20170925202928',
- 'schema-salad==2.6.20170925203755',
+ 'cwltool==1.0.20170928192020',
+ 'schema-salad==2.6.20170927145003',
'typing==3.5.3.0',
'ruamel.yaml==0.13.7',
'arvados-python-client>=0.1.20170526013812',
commit 38fa977846f51e07db0e77c4270c0fb9e8e2fe5c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Sep 26 15:49:43 2017 -0400
12278: Filter nameext and nameroot from final cwl.output.json
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 044b490..5756789 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -281,7 +281,7 @@ class ArvCwlRunner(object):
def rewrite(fileobj):
fileobj["location"] = generatemapper.mapper(fileobj["location"]).target
- for k in ("basename", "listing", "contents"):
+ for k in ("basename", "listing", "contents", "nameext", "nameroot", "dirname"):
if k in fileobj:
del fileobj[k]
commit 1275b20bc802424a69abd8a76b1c9e3de9657419
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Sep 26 09:44:57 2017 -0400
12278: SourceLine yields stack trace if debugging enabled
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7f4b5c7..044b490 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -237,7 +237,7 @@ class ArvCwlRunner(object):
self.check_features(v)
elif isinstance(obj, list):
for i,v in enumerate(obj):
- with SourceLine(obj, i, UnsupportedRequirement):
+ with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
self.check_features(v)
def make_output_collection(self, name, tagsString, outputObj):
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index 0513ca0..e59903f 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -33,7 +33,7 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
if dockerRequirement["dockerImageId"] in cached_lookups:
return dockerRequirement["dockerImageId"]
- with SourceLine(dockerRequirement, "dockerImageId", WorkflowException):
+ with SourceLine(dockerRequirement, "dockerImageId", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
sp = dockerRequirement["dockerImageId"].split(":")
image_name = sp[0]
image_tag = sp[1] if len(sp) > 1 else "latest"
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 20cb467..fdf506e 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -83,7 +83,7 @@ class ArvadosWorkflow(Workflow):
kwargs["work_api"] = self.work_api
req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
if req:
- with SourceLine(self.tool, None, WorkflowException):
+ with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
if "id" not in self.tool:
raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
document_loader, workflowobj, uri = (self.doc_loader, self.doc_loader.fetch(self.tool["id"]), self.tool["id"])
@@ -114,10 +114,10 @@ class ArvadosWorkflow(Workflow):
def keepmount(obj):
remove_redundant_fields(obj)
- with SourceLine(obj, None, WorkflowException):
+ with SourceLine(obj, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
if "location" not in obj:
raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
- with SourceLine(obj, "location", WorkflowException):
+ with SourceLine(obj, "location", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
if obj["location"].startswith("keep:"):
obj["location"] = "/keep/" + obj["location"][5:]
if "listing" in obj:
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index c8ae77a..213fd21 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -58,6 +58,8 @@ class ArvPathMapper(PathMapper):
if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True)
+ debug = logger.isEnabledFor(logging.DEBUG)
+
if src not in self._pathmap:
if src.startswith("file:"):
# Local FS ref, may need to be uploaded or may be on keep
@@ -67,7 +69,7 @@ class ArvPathMapper(PathMapper):
fnPattern="keep:%s/%s",
dirPattern="keep:%s/%s",
raiseOSError=True)
- with SourceLine(srcobj, "location", WorkflowException):
+ with SourceLine(srcobj, "location", WorkflowException, debug):
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
@@ -82,10 +84,10 @@ class ArvPathMapper(PathMapper):
else:
self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
- with SourceLine(srcobj, "secondaryFiles", WorkflowException):
+ with SourceLine(srcobj, "secondaryFiles", WorkflowException, debug):
for l in srcobj.get("secondaryFiles", []):
self.visit(l, uploadfiles)
- with SourceLine(srcobj, "listing", WorkflowException):
+ with SourceLine(srcobj, "listing", WorkflowException, debug):
for l in srcobj.get("listing", []):
self.visit(l, uploadfiles)
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 49ab07d..813feed 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -51,8 +51,8 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20170925201615',
- 'schema-salad==2.6.20170925201518',
+ 'cwltool==1.0.20170925202928',
+ 'schema-salad==2.6.20170925203755',
'typing==3.5.3.0',
'ruamel.yaml==0.13.7',
'arvados-python-client>=0.1.20170526013812',
commit 09492ef241af355eea0ef1cb850d9d06ec4e7ca4
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Sep 25 16:10:39 2017 -0400
12278: Can provide custom schema salad to dev jobs build.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index 46e525b..639096c 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -15,6 +15,7 @@ Syntax:
WORKSPACE=path Path to the Arvados source tree to build packages from
CWLTOOL=path (optional) Path to cwltool git repository.
+SALAD=path (optional) Path to schema_salad git repository.
EOF
@@ -42,6 +43,14 @@ sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
(cd sdk/cwl && python setup.py sdist)
runner=$(cd sdk/cwl/dist && ls -t arvados-cwl-runner-*.tar.gz | head -n1)
+rm -rf sdk/cwl/salad_dist
+mkdir -p sdk/cwl/salad_dist
+if [[ -n "$SALAD" ]] ; then
+ (cd "$SALAD" && python setup.py sdist)
+ salad=$(cd "$SALAD/dist" && ls -t schema-salad-*.tar.gz | head -n1)
+ cp "$SALAD/dist/$salad" $WORKSPACE/sdk/cwl/salad_dist
+fi
+
rm -rf sdk/cwl/cwltool_dist
mkdir -p sdk/cwl/cwltool_dist
if [[ -n "$CWLTOOL" ]] ; then
@@ -61,6 +70,6 @@ else
gittag=$(git log --first-parent --max-count=1 --format=format:%H sdk/cwl)
fi
-docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg cwltool=$cwltool -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$gittag "$WORKSPACE/sdk"
+docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$gittag "$WORKSPACE/sdk"
echo arv-keepdocker arvados/jobs $gittag
arv-keepdocker arvados/jobs $gittag
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 50f9cf4..49ab07d 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -51,8 +51,8 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20170828135420',
- 'schema-salad==2.6.20170712194300',
+ 'cwltool==1.0.20170925201615',
+ 'schema-salad==2.6.20170925201518',
'typing==3.5.3.0',
'ruamel.yaml==0.13.7',
'arvados-python-client>=0.1.20170526013812',
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index cc08ad7..f9f1e96 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -24,13 +24,16 @@ RUN pip install -U setuptools
ARG sdk
ARG runner
+ARG salad
ARG cwltool
ADD python/dist/$sdk /tmp/
+ADD cwl/salad_dist/$salad /tmp/
ADD cwl/cwltool_dist/$cwltool /tmp/
ADD cwl/dist/$runner /tmp/
RUN cd /tmp/arvados-python-client-* && python setup.py install
+RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && python setup.py install ; fi
RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && python setup.py install ; fi
RUN cd /tmp/arvados-cwl-runner-* && python setup.py install
commit a355b779c5d8df691b726ba18dd88892097017ae
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Sep 25 11:32:20 2017 -0400
12278: Propagate --debug flag to jobs.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 794c994..7fc62db 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -297,6 +297,9 @@ class RunnerJob(Runner):
if self.on_error:
self.job_order["arv:on_error"] = self.on_error
+ if kwargs.get("debug"):
+ self.job_order["arv:debug"] = True
+
return {
"script": "cwl-runner",
"script_version": "master",
diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 7fbbd29..80235e4 100644
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -75,6 +75,8 @@ def run():
output_tags = None
enable_reuse = True
on_error = "continue"
+ debug = False
+
if "arv:output_name" in job_order_object:
output_name = job_order_object["arv:output_name"]
del job_order_object["arv:output_name"]
@@ -91,6 +93,10 @@ def run():
on_error = job_order_object["arv:on_error"]
del job_order_object["arv:on_error"]
+ if "arv:debug" in job_order_object:
+ debug = job_order_object["arv:debug"]
+ del job_order_object["arv:debug"]
+
runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
output_name=output_name, output_tags=output_tags)
@@ -108,7 +114,7 @@ def run():
args.enable_reuse = enable_reuse
args.on_error = on_error
args.submit = False
- args.debug = False
+ args.debug = debug
args.quiet = False
args.ignore_docker_for_reuse = False
args.basedir = os.getcwd()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list