[ARVADOS] created: 0cdbfaa368e00fe2bbc026f55e1027f203284979

Git user git at public.curoverse.com
Thu Sep 28 15:24:40 EDT 2017


        at  0cdbfaa368e00fe2bbc026f55e1027f203284979 (commit)


commit 0cdbfaa368e00fe2bbc026f55e1027f203284979
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Sep 28 12:52:44 2017 -0400

    12278: Need to set debug logging in crunch_script.py
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 80235e4..12d74a0 100644
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -109,6 +109,11 @@ def run():
                                                   fs_access=make_fs_access(""),
                                                   num_retries=runner.num_retries))
 
+        if debug:
+            logger.setLevel(logging.DEBUG)
+            logging.getLogger('arvados').setLevel(logging.DEBUG)
+            logging.getLogger("cwltool").setLevel(logging.DEBUG)
+
         args = argparse.Namespace()
         args.project_uuid = arvados.current_job()["owner_uuid"]
         args.enable_reuse = enable_reuse

commit 8a9be3ff4744d82018cce0cddf7f97b7cfe86a40
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Sep 28 12:12:33 2017 -0400

    12278: Bump schema-salad and cwltool for SourceLine fix.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 813feed..762c6fd 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -51,8 +51,8 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20170925202928',
-          'schema-salad==2.6.20170925203755',
+          'cwltool==1.0.20170928192020',
+          'schema-salad==2.6.20170927145003',
           'typing==3.5.3.0',
           'ruamel.yaml==0.13.7',
           'arvados-python-client>=0.1.20170526013812',

commit 38fa977846f51e07db0e77c4270c0fb9e8e2fe5c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Sep 26 15:49:43 2017 -0400

    12278: Filter nameext and nameroot from final cwl.output.json
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 044b490..5756789 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -281,7 +281,7 @@ class ArvCwlRunner(object):
 
         def rewrite(fileobj):
             fileobj["location"] = generatemapper.mapper(fileobj["location"]).target
-            for k in ("basename", "listing", "contents"):
+            for k in ("basename", "listing", "contents", "nameext", "nameroot", "dirname"):
                 if k in fileobj:
                     del fileobj[k]
 

commit 1275b20bc802424a69abd8a76b1c9e3de9657419
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Sep 26 09:44:57 2017 -0400

    12278: SourceLine yields stack trace if debugging enabled
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7f4b5c7..044b490 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -237,7 +237,7 @@ class ArvCwlRunner(object):
                 self.check_features(v)
         elif isinstance(obj, list):
             for i,v in enumerate(obj):
-                with SourceLine(obj, i, UnsupportedRequirement):
+                with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
                     self.check_features(v)
 
     def make_output_collection(self, name, tagsString, outputObj):
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index 0513ca0..e59903f 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -33,7 +33,7 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
         if dockerRequirement["dockerImageId"] in cached_lookups:
             return dockerRequirement["dockerImageId"]
 
-    with SourceLine(dockerRequirement, "dockerImageId", WorkflowException):
+    with SourceLine(dockerRequirement, "dockerImageId", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
         sp = dockerRequirement["dockerImageId"].split(":")
         image_name = sp[0]
         image_tag = sp[1] if len(sp) > 1 else "latest"
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 20cb467..fdf506e 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -83,7 +83,7 @@ class ArvadosWorkflow(Workflow):
         kwargs["work_api"] = self.work_api
         req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
         if req:
-            with SourceLine(self.tool, None, WorkflowException):
+            with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
                 if "id" not in self.tool:
                     raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
             document_loader, workflowobj, uri = (self.doc_loader, self.doc_loader.fetch(self.tool["id"]), self.tool["id"])
@@ -114,10 +114,10 @@ class ArvadosWorkflow(Workflow):
 
                 def keepmount(obj):
                     remove_redundant_fields(obj)
-                    with SourceLine(obj, None, WorkflowException):
+                    with SourceLine(obj, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
                         if "location" not in obj:
                             raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
-                    with SourceLine(obj, "location", WorkflowException):
+                    with SourceLine(obj, "location", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
                         if obj["location"].startswith("keep:"):
                             obj["location"] = "/keep/" + obj["location"][5:]
                             if "listing" in obj:
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index c8ae77a..213fd21 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -58,6 +58,8 @@ class ArvPathMapper(PathMapper):
         if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
             self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True)
 
+        debug = logger.isEnabledFor(logging.DEBUG)
+
         if src not in self._pathmap:
             if src.startswith("file:"):
                 # Local FS ref, may need to be uploaded or may be on keep
@@ -67,7 +69,7 @@ class ArvPathMapper(PathMapper):
                                                    fnPattern="keep:%s/%s",
                                                    dirPattern="keep:%s/%s",
                                                    raiseOSError=True)
-                with SourceLine(srcobj, "location", WorkflowException):
+                with SourceLine(srcobj, "location", WorkflowException, debug):
                     if isinstance(st, arvados.commands.run.UploadFile):
                         uploadfiles.add((src, ab, st))
                     elif isinstance(st, arvados.commands.run.ArvFile):
@@ -82,10 +84,10 @@ class ArvPathMapper(PathMapper):
             else:
                 self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
 
-        with SourceLine(srcobj, "secondaryFiles", WorkflowException):
+        with SourceLine(srcobj, "secondaryFiles", WorkflowException, debug):
             for l in srcobj.get("secondaryFiles", []):
                 self.visit(l, uploadfiles)
-        with SourceLine(srcobj, "listing", WorkflowException):
+        with SourceLine(srcobj, "listing", WorkflowException, debug):
             for l in srcobj.get("listing", []):
                 self.visit(l, uploadfiles)
 
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 49ab07d..813feed 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -51,8 +51,8 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20170925201615',
-          'schema-salad==2.6.20170925201518',
+          'cwltool==1.0.20170925202928',
+          'schema-salad==2.6.20170925203755',
           'typing==3.5.3.0',
           'ruamel.yaml==0.13.7',
           'arvados-python-client>=0.1.20170526013812',

commit 09492ef241af355eea0ef1cb850d9d06ec4e7ca4
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Sep 25 16:10:39 2017 -0400

    12278: Can provide custom schema salad to dev jobs build.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index 46e525b..639096c 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -15,6 +15,7 @@ Syntax:
 
 WORKSPACE=path         Path to the Arvados source tree to build packages from
 CWLTOOL=path           (optional) Path to cwltool git repository.
+SALAD=path             (optional) Path to schema_salad git repository.
 
 EOF
 
@@ -42,6 +43,14 @@ sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
 (cd sdk/cwl && python setup.py sdist)
 runner=$(cd sdk/cwl/dist && ls -t arvados-cwl-runner-*.tar.gz | head -n1)
 
+rm -rf sdk/cwl/salad_dist
+mkdir -p sdk/cwl/salad_dist
+if [[ -n "$SALAD" ]] ; then
+    (cd "$SALAD" && python setup.py sdist)
+    salad=$(cd "$SALAD/dist" && ls -t schema-salad-*.tar.gz | head -n1)
+    cp "$SALAD/dist/$salad" $WORKSPACE/sdk/cwl/salad_dist
+fi
+
 rm -rf sdk/cwl/cwltool_dist
 mkdir -p sdk/cwl/cwltool_dist
 if [[ -n "$CWLTOOL" ]] ; then
@@ -61,6 +70,6 @@ else
     gittag=$(git log --first-parent --max-count=1 --format=format:%H sdk/cwl)
 fi
 
-docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg cwltool=$cwltool -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$gittag "$WORKSPACE/sdk"
+docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$gittag "$WORKSPACE/sdk"
 echo arv-keepdocker arvados/jobs $gittag
 arv-keepdocker arvados/jobs $gittag
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 50f9cf4..49ab07d 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -51,8 +51,8 @@ setup(name='arvados-cwl-runner',
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to build.
       install_requires=[
-          'cwltool==1.0.20170828135420',
-          'schema-salad==2.6.20170712194300',
+          'cwltool==1.0.20170925201615',
+          'schema-salad==2.6.20170925201518',
           'typing==3.5.3.0',
           'ruamel.yaml==0.13.7',
           'arvados-python-client>=0.1.20170526013812',
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index cc08ad7..f9f1e96 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -24,13 +24,16 @@ RUN pip install -U setuptools
 
 ARG sdk
 ARG runner
+ARG salad
 ARG cwltool
 
 ADD python/dist/$sdk /tmp/
+ADD cwl/salad_dist/$salad /tmp/
 ADD cwl/cwltool_dist/$cwltool /tmp/
 ADD cwl/dist/$runner /tmp/
 
 RUN cd /tmp/arvados-python-client-* && python setup.py install
+RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && python setup.py install ; fi
 RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && python setup.py install ; fi
 RUN cd /tmp/arvados-cwl-runner-* && python setup.py install
 

commit a355b779c5d8df691b726ba18dd88892097017ae
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Sep 25 11:32:20 2017 -0400

    12278: Propagate --debug flag to jobs.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 794c994..7fc62db 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -297,6 +297,9 @@ class RunnerJob(Runner):
         if self.on_error:
             self.job_order["arv:on_error"] = self.on_error
 
+        if kwargs.get("debug"):
+            self.job_order["arv:debug"] = True
+
         return {
             "script": "cwl-runner",
             "script_version": "master",
diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 7fbbd29..80235e4 100644
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -75,6 +75,8 @@ def run():
         output_tags = None
         enable_reuse = True
         on_error = "continue"
+        debug = False
+
         if "arv:output_name" in job_order_object:
             output_name = job_order_object["arv:output_name"]
             del job_order_object["arv:output_name"]
@@ -91,6 +93,10 @@ def run():
             on_error = job_order_object["arv:on_error"]
             del job_order_object["arv:on_error"]
 
+        if "arv:debug" in job_order_object:
+            debug = job_order_object["arv:debug"]
+            del job_order_object["arv:debug"]
+
         runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
                                           output_name=output_name, output_tags=output_tags)
 
@@ -108,7 +114,7 @@ def run():
         args.enable_reuse = enable_reuse
         args.on_error = on_error
         args.submit = False
-        args.debug = False
+        args.debug = debug
         args.quiet = False
         args.ignore_docker_for_reuse = False
         args.basedir = os.getcwd()

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list