[ARVADOS] created: 1.3.0-853-gbfe31b5db
Git user
git at public.curoverse.com
Fri May 3 23:29:30 UTC 2019
at bfe31b5db42a7c66b49ebcd519e90d590cafe825 (commit)
commit bfe31b5db42a7c66b49ebcd519e90d590cafe825
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri May 3 19:06:02 2019 -0400
15028: Support WorkReuse
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 0c7ad732d..e20eb18dd 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -276,6 +276,9 @@ class ArvadosContainer(JobBase):
enable_reuse = runtimeContext.enable_reuse
if enable_reuse:
+ reuse_req, _ = self.get_requirement("WorkReuse")
+ if reuse_req:
+ enable_reuse = reuse_req["enableReuse"]
reuse_req, _ = self.get_requirement("http://arvados.org/cwl#ReuseRequirement")
if reuse_req:
enable_reuse = reuse_req["enableReuse"]
commit 56555817311d4cadf261cd3aa380361690e6fe65
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri May 3 16:43:41 2019 -0400
15028: Fix timelimit, set_secondaryfiles
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 860bd954c..0c7ad732d 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -260,7 +260,7 @@ class ArvadosContainer(JobBase):
if self.output_ttl < 0:
raise WorkflowException("Invalid value %d for output_ttl, cannot be less than zero" % container_request["output_ttl"])
- if self.timelimit is not None:
+ if self.timelimit is not None and self.timelimit > 0:
scheduling_parameters["max_run_time"] = self.timelimit
extra_submit_params = {}
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 68b2fd827..3a3784050 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -137,7 +137,8 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
# set secondaryFiles, may be inherited by compound types.
secondaryspec = inputschema["secondaryFiles"]
- if isinstance(inputschema["type"], (Mapping, Sequence)):
+ if (isinstance(inputschema["type"], (Mapping, Sequence)) and
+ not isinstance(inputschema["type"], basestring)):
# compound type (union, array, record)
set_secondary(fsaccess, builder, inputschema["type"], secondaryspec, primary, discovered)
commit 35852356dd9bb2e29e8aebb208e063d9fe924316
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri May 3 15:29:16 2019 -0400
15028: Fix set_secondary with union types
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 752cc32ba..68b2fd827 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -137,8 +137,8 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
# set secondaryFiles, may be inherited by compound types.
secondaryspec = inputschema["secondaryFiles"]
- if isinstance(inputschema["type"], Mapping):
- # compound type (array or record)
+ if isinstance(inputschema["type"], (Mapping, Sequence)):
+ # compound type (union, array, record)
set_secondary(fsaccess, builder, inputschema["type"], secondaryspec, primary, discovered)
elif (inputschema["type"] == "record" and
commit c042cd8120be2c54f54ee2a6041adbc30e141190
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri May 3 14:32:21 2019 -0400
15028: More conformance fixes
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index def720381..f8069a3cb 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -318,6 +318,15 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
else:
arvados.log_handler.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s'))
+ if stdout is sys.stdout:
+ # cwltool.main has code to work around encoding issues with
+ # sys.stdout and unix pipes (they default to ASCII encoding,
+ # we want utf-8), so when stdout is sys.stdout set it to None
+ # to take advantage of that. Don't override it for all cases
+ # since we still want to be able to capture stdout for the
+ # unit tests.
+ stdout = None
+
return cwltool.main.main(args=arvargs,
stdout=stdout,
stderr=stderr,
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index b0dff491f..860bd954c 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -225,6 +225,10 @@ class ArvadosContainer(JobBase):
runtimeContext.pull_image,
runtimeContext.project_uuid)
+ network_req, _ = self.get_requirement("NetworkAccess")
+ if network_req:
+ runtime_constraints["API"] = network_req["networkAccess"]
+
api_req, _ = self.get_requirement("http://arvados.org/cwl#APIRequirement")
if api_req:
runtime_constraints["API"] = True
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 9b9c2f161..9ff06fa4a 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -6,7 +6,7 @@ from __future__ import division
from builtins import next
from builtins import object
from builtins import str
-from future.utils import viewvalues
+from future.utils import viewvalues, viewitems
import argparse
import logging
@@ -406,7 +406,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
except (KeyboardInterrupt, SystemExit):
break
- def check_features(self, obj):
+ def check_features(self, obj, parentfield=""):
if isinstance(obj, dict):
if obj.get("writable") and self.work_api != "containers":
raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported with --api=jobs")
@@ -420,12 +420,15 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
"Option 'dockerOutputDirectory' must be an absolute path.")
if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers":
raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs")
- for v in viewvalues(obj):
- self.check_features(v)
+ if obj.get("class") == "InplaceUpdateRequirement":
+ if obj["inplaceUpdate"] and parentfield == "requirements":
+ raise SourceLine(obj, "class", UnsupportedRequirement).makeError("InplaceUpdateRequirement not supported for keep collections.")
+ for k,v in viewitems(obj):
+ self.check_features(v, parentfield=k)
elif isinstance(obj, list):
for i,v in enumerate(obj):
with SourceLine(obj, i, UnsupportedRequirement, logger.isEnabledFor(logging.DEBUG)):
- self.check_features(v)
+ self.check_features(v, parentfield=parentfield)
def make_output_collection(self, name, storage_classes, tagsString, outputObj):
outputObj = copy.deepcopy(outputObj)
@@ -525,6 +528,18 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
'progress':1.0
}).execute(num_retries=self.num_retries)
+ def apply_reqs(self, job_order_object, tool):
+ if "https://w3id.org/cwl/cwl#requirements" in job_order_object:
+ if tool.metadata.get("http://commonwl.org/cwltool#original_cwlVersion") == 'v1.0':
+ raise WorkflowException(
+ "`cwl:requirements` in the input object is not part of CWL "
+ "v1.0. You can adjust to use `cwltool:overrides` instead; or you "
+ "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with "
+ "--enable-dev.")
+ job_reqs = job_order_object["https://w3id.org/cwl/cwl#requirements"]
+ for req in job_reqs:
+ tool.requirements.append(req)
+
def arv_executor(self, tool, job_order, runtimeContext, logger=None):
self.debug = runtimeContext.debug
@@ -607,6 +622,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
merged_map=merged_map),
"success")
+ self.apply_reqs(job_order, tool)
+
self.ignore_docker_for_reuse = runtimeContext.ignore_docker_for_reuse
self.eval_timeout = runtimeContext.eval_timeout
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 3af3dfb4d..def79ece4 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20190502174223',
+ 'cwltool==1.0.20190503175737',
'schema-salad==4.1.20190305210046',
'typing >= 3.6.4',
'ruamel.yaml >=0.15.54, <=0.15.77',
commit 420b31d5f181d320e6f770cc3c63f2ae66908c88
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu May 2 17:20:10 2019 -0400
15028: Handling secondaryFiles on record fields
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 4a8a22a4a..752cc32ba 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -5,6 +5,7 @@
from future import standard_library
standard_library.install_aliases()
from future.utils import viewvalues, viewitems
+from past.builtins import basestring
import os
import sys
@@ -106,36 +107,91 @@ def make_builder(joborder, hints, requirements, runtimeContext):
stagedir="", # type: Text
)
+def search_schemadef(name, reqs):
+ for r in reqs:
+ if r["class"] == "SchemaDefRequirement":
+ for sd in r["types"]:
+ if sd["name"] == name:
+ return sd
+ return None
+
+primitive_types_set = frozenset(("null", "boolean", "int", "long",
+ "float", "double", "string", "record",
+ "array", "enum"))
+
+def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discovered):
+ if isinstance(inputschema, Sequence) and not isinstance(inputschema, basestring):
+ # union type, collect all possible secondaryFiles
+ for i in inputschema:
+ set_secondary(fsaccess, builder, i, secondaryspec, primary, discovered)
+ return
+
+ if isinstance(inputschema, basestring):
+ sd = search_schemadef(inputschema, reversed(builder.hints+builder.requirements))
+ if sd:
+ inputschema = sd
+ else:
+ return
-def set_secondary(fsaccess, builder, inputschema, primary, discovered):
- if isinstance(primary, Mapping) and primary.get("class") == "File":
- if "secondaryFiles" not in primary:
- primary["secondaryFiles"] = []
- for i, sf in enumerate(inputschema["secondaryFiles"]):
- pattern = builder.do_eval(sf["pattern"], context=primary)
- if pattern is None:
- continue
- sfpath = substitute(primary["location"], pattern)
- required = builder.do_eval(sf["required"], context=primary)
-
- if fsaccess.exists(sfpath):
- primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
- elif required:
- raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
- "Required secondary file '%s' does not exist" % sfpath)
-
- primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
- if discovered is not None:
- discovered[primary["location"]] = primary["secondaryFiles"]
- elif isinstance(primary, Sequence):
- for e in primary:
- set_secondary(fsaccess, builder, inputschema, e, discovered)
+ if "secondaryFiles" in inputschema:
+ # set secondaryFiles, may be inherited by compound types.
+ secondaryspec = inputschema["secondaryFiles"]
+
+ if isinstance(inputschema["type"], Mapping):
+ # compound type (array or record)
+ set_secondary(fsaccess, builder, inputschema["type"], secondaryspec, primary, discovered)
+
+ elif (inputschema["type"] == "record" and
+ isinstance(primary, Mapping)):
+ #
+ # record type, find secondary files associated with fields.
+ #
+ for f in inputschema["fields"]:
+ p = primary.get(shortname(f["name"]))
+ if p:
+ set_secondary(fsaccess, builder, f, secondaryspec, p, discovered)
+
+ elif (inputschema["type"] == "array" and
+ isinstance(primary, Sequence)):
+ #
+ # array type, find secondary files of elements
+ #
+ for p in primary:
+ set_secondary(fsaccess, builder, {"type": inputschema["items"]}, secondaryspec, p, discovered)
+
+ elif (inputschema["type"] == "File" and
+ secondaryspec and
+ isinstance(primary, Mapping) and
+ primary.get("class") == "File" and
+ "secondaryFiles" not in primary):
+ #
+ # Found a file, check for secondaryFiles
+ #
+ primary["secondaryFiles"] = []
+ for i, sf in enumerate(aslist(secondaryspec)):
+ pattern = builder.do_eval(sf["pattern"], context=primary)
+ if pattern is None:
+ continue
+ sfpath = substitute(primary["location"], pattern)
+ required = builder.do_eval(sf["required"], context=primary)
+
+ if fsaccess.exists(sfpath):
+ primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
+ elif required:
+ raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+ "Required secondary file '%s' does not exist" % sfpath)
+
+ primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
+ if discovered is not None:
+ discovered[primary["location"]] = primary["secondaryFiles"]
+ elif inputschema["type"] not in primitive_types_set:
+ set_secondary(fsaccess, builder, inputschema["type"], secondaryspec, primary, discovered)
def discover_secondary_files(fsaccess, builder, inputs, job_order, discovered=None):
for inputschema in inputs:
primary = job_order.get(shortname(inputschema["id"]))
- if isinstance(primary, (Mapping, Sequence)) and inputschema.get("secondaryFiles"):
- set_secondary(fsaccess, builder, inputschema, primary, discovered)
+ if isinstance(primary, (Mapping, Sequence)):
+ set_secondary(fsaccess, builder, inputschema, None, primary, discovered)
collection_uuid_pattern = re.compile(r'^keep:([a-z0-9]{5}-4zz18-[a-z0-9]{15})(/.*)?$')
collection_pdh_pattern = re.compile(r'^keep:([0-9a-f]{32}\+\d+)(/.*)?')
@@ -418,8 +474,8 @@ def upload_job_order(arvrunner, name, tool, job_order):
arvrunner.fs_access)
# Need to create a builder object to evaluate expressions.
builder = make_builder(builder_job_order,
- tool.tool.get("hints", []),
- tool.tool.get("requirements", []),
+ tool.hints,
+ tool.requirements,
ArvRuntimeContext())
# Now update job_order with secondaryFiles
discover_secondary_files(arvrunner.fs_access,
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index c9776eac1..3af3dfb4d 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20190426195829',
+ 'cwltool==1.0.20190502174223',
'schema-salad==4.1.20190305210046',
'typing >= 3.6.4',
'ruamel.yaml >=0.15.54, <=0.15.77',
commit 1746985ec50d72cad304640f1ea39e2a9dff07bf
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu May 2 15:25:40 2019 -0400
15028: Support optional secondaryFiles
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 31e6be12b..4fc02a016 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -3,10 +3,10 @@
# SPDX-License-Identifier: Apache-2.0
from cwltool.command_line_tool import CommandLineTool, ExpressionTool
-from cwltool.builder import Builder
from .arvjob import ArvadosJob
from .arvcontainer import ArvadosContainer
from .pathmapper import ArvPathMapper
+from .runner import make_builder
from functools import partial
from schema_salad.sourceline import SourceLine
from cwltool.errors import WorkflowException
@@ -37,30 +37,6 @@ def set_cluster_target(tool, arvrunner, builder, runtimeContext):
return runtimeContext
-def make_builder(joborder, hints, requirements, runtimeContext):
- return Builder(
- job=joborder,
- files=[], # type: List[Dict[Text, Text]]
- bindings=[], # type: List[Dict[Text, Any]]
- schemaDefs={}, # type: Dict[Text, Dict[Text, Any]]
- names=None, # type: Names
- requirements=requirements, # type: List[Dict[Text, Any]]
- hints=hints, # type: List[Dict[Text, Any]]
- resources={}, # type: Dict[str, int]
- mutation_manager=None, # type: Optional[MutationManager]
- formatgraph=None, # type: Optional[Graph]
- make_fs_access=None, # type: Type[StdFsAccess]
- fs_access=None, # type: StdFsAccess
- job_script_provider=runtimeContext.job_script_provider, # type: Optional[Any]
- timeout=runtimeContext.eval_timeout, # type: float
- debug=runtimeContext.debug, # type: bool
- js_console=runtimeContext.js_console, # type: bool
- force_docker_pull=runtimeContext.force_docker_pull, # type: bool
- loadListing="", # type: Text
- outdir="", # type: Text
- tmpdir="", # type: Text
- stagedir="", # type: Text
- )
class ArvadosCommandTool(CommandLineTool):
"""Wrap cwltool CommandLineTool to override selected methods."""
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 325fccb2c..3c60ac9fd 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -22,9 +22,11 @@ from cwltool.context import LoadingContext
import ruamel.yaml as yaml
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
- trim_anonymous_location, remove_redundant_fields, discover_secondary_files)
+ trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
+ make_builder)
from .pathmapper import ArvPathMapper, trim_listing
-from .arvtool import ArvadosCommandTool, set_cluster_target, make_builder
+from .arvtool import ArvadosCommandTool, set_cluster_target
+
from .perf import Perf
logger = logging.getLogger('arvados.cwl-runner')
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index ed6bcd008..4a8a22a4a 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -13,8 +13,10 @@ import urllib.parse
from functools import partial
import logging
import json
+import copy
from collections import namedtuple
from io import StringIO
+from typing import Mapping, Sequence
if os.name == "posix" and sys.version_info[0] < 3:
import subprocess32 as subprocess
@@ -25,13 +27,15 @@ from schema_salad.sourceline import SourceLine, cmap
from cwltool.command_line_tool import CommandLineTool
import cwltool.workflow
-from cwltool.process import scandeps, UnsupportedRequirement, normalizeFilesDirs, shortname, Process
+from cwltool.process import (scandeps, UnsupportedRequirement, normalizeFilesDirs,
+ shortname, Process, fill_in_defaults)
from cwltool.load_tool import fetch_document
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
from cwltool.utils import aslist
from cwltool.builder import substitute
from cwltool.pack import pack
from cwltool.update import INTERNAL_VERSION
+from cwltool.builder import Builder
import schema_salad.validate as validate
import arvados.collection
@@ -42,6 +46,7 @@ import arvados_cwl.arvdocker
from .pathmapper import ArvPathMapper, trim_listing
from ._version import __version__
from . import done
+from . context import ArvRuntimeContext
logger = logging.getLogger('arvados.cwl-runner')
@@ -76,20 +81,61 @@ def find_defaults(d, op):
for i in viewvalues(d):
find_defaults(i, op)
-def setSecondary(t, fileobj, discovered):
- if isinstance(fileobj, dict) and fileobj.get("class") == "File":
- if "secondaryFiles" not in fileobj:
- fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf["pattern"]), "class": "File"} for sf in t["secondaryFiles"]])
+def make_builder(joborder, hints, requirements, runtimeContext):
+ return Builder(
+ job=joborder,
+ files=[], # type: List[Dict[Text, Text]]
+ bindings=[], # type: List[Dict[Text, Any]]
+ schemaDefs={}, # type: Dict[Text, Dict[Text, Any]]
+ names=None, # type: Names
+ requirements=requirements, # type: List[Dict[Text, Any]]
+ hints=hints, # type: List[Dict[Text, Any]]
+ resources={}, # type: Dict[str, int]
+ mutation_manager=None, # type: Optional[MutationManager]
+ formatgraph=None, # type: Optional[Graph]
+ make_fs_access=None, # type: Type[StdFsAccess]
+ fs_access=None, # type: StdFsAccess
+ job_script_provider=runtimeContext.job_script_provider, # type: Optional[Any]
+ timeout=runtimeContext.eval_timeout, # type: float
+ debug=runtimeContext.debug, # type: bool
+ js_console=runtimeContext.js_console, # type: bool
+ force_docker_pull=runtimeContext.force_docker_pull, # type: bool
+ loadListing="", # type: Text
+ outdir="", # type: Text
+ tmpdir="", # type: Text
+ stagedir="", # type: Text
+ )
+
+
+def set_secondary(fsaccess, builder, inputschema, primary, discovered):
+ if isinstance(primary, Mapping) and primary.get("class") == "File":
+ if "secondaryFiles" not in primary:
+ primary["secondaryFiles"] = []
+ for i, sf in enumerate(inputschema["secondaryFiles"]):
+ pattern = builder.do_eval(sf["pattern"], context=primary)
+ if pattern is None:
+ continue
+ sfpath = substitute(primary["location"], pattern)
+ required = builder.do_eval(sf["required"], context=primary)
+
+ if fsaccess.exists(sfpath):
+ primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
+ elif required:
+ raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+ "Required secondary file '%s' does not exist" % sfpath)
+
+ primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
if discovered is not None:
- discovered[fileobj["location"]] = fileobj["secondaryFiles"]
- elif isinstance(fileobj, list):
- for e in fileobj:
- setSecondary(t, e, discovered)
+ discovered[primary["location"]] = primary["secondaryFiles"]
+ elif isinstance(primary, Sequence):
+ for e in primary:
+ set_secondary(fsaccess, builder, inputschema, e, discovered)
-def discover_secondary_files(inputs, job_order, discovered=None):
- for t in inputs:
- if shortname(t["id"]) in job_order and t.get("secondaryFiles"):
- setSecondary(t, job_order[shortname(t["id"])], discovered)
+def discover_secondary_files(fsaccess, builder, inputs, job_order, discovered=None):
+ for inputschema in inputs:
+ primary = job_order.get(shortname(inputschema["id"]))
+ if isinstance(primary, (Mapping, Sequence)) and inputschema.get("secondaryFiles"):
+ set_secondary(fsaccess, builder, inputschema, primary, discovered)
collection_uuid_pattern = re.compile(r'^keep:([a-z0-9]{5}-4zz18-[a-z0-9]{15})(/.*)?$')
collection_pdh_pattern = re.compile(r'^keep:([0-9a-f]{32}\+\d+)(/.*)?')
@@ -220,8 +266,18 @@ def upload_dependencies(arvrunner, name, document_loader,
discovered = {}
def discover_default_secondary_files(obj):
- discover_secondary_files(obj["inputs"],
- {shortname(t["id"]): t["default"] for t in obj["inputs"] if "default" in t},
+ builder_job_order = {}
+ for t in obj["inputs"]:
+ builder_job_order[shortname(t["id"])] = t["default"] if "default" in t else None
+ # Need to create a builder object to evaluate expressions.
+ builder = make_builder(builder_job_order,
+ obj.get("hints", []),
+ obj.get("requirements", []),
+ ArvRuntimeContext())
+ discover_secondary_files(arvrunner.fs_access,
+ builder,
+ obj["inputs"],
+ builder_job_order,
discovered)
visit_class(workflowobj, ("CommandLineTool", "Workflow"), discover_default_secondary_files)
@@ -355,7 +411,21 @@ def upload_job_order(arvrunner, name, tool, job_order):
object with 'location' updated to the proper keep references.
"""
- discover_secondary_files(tool.tool["inputs"], job_order)
+ # Make a copy of the job order and set defaults.
+ builder_job_order = copy.copy(job_order)
+ fill_in_defaults(tool.tool["inputs"],
+ builder_job_order,
+ arvrunner.fs_access)
+ # Need to create a builder object to evaluate expressions.
+ builder = make_builder(builder_job_order,
+ tool.tool.get("hints", []),
+ tool.tool.get("requirements", []),
+ ArvRuntimeContext())
+ # Now update job_order with secondaryFiles
+ discover_secondary_files(arvrunner.fs_access,
+ builder,
+ tool.tool["inputs"],
+ job_order)
jobmapper = upload_dependencies(arvrunner,
name,
commit 21b517dfc95984e812b4b9c4243edfb0dbf5754a
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Apr 30 14:18:24 2019 -0400
15028: Bump cwltool
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 2b273017e..c9776eac1 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20190426175817',
+ 'cwltool==1.0.20190426195829',
'schema-salad==4.1.20190305210046',
'typing >= 3.6.4',
'ruamel.yaml >=0.15.54, <=0.15.77',
commit a3b40d4546cb18744d6052ffb47f5f81773ac261
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Apr 29 15:40:24 2019 -0400
15028: Propagate --enable-dev and handle blank node ids
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 437640ac6..b0dff491f 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -485,6 +485,9 @@ class RunnerContainer(Runner):
if self.arvrunner.project_uuid:
command.append("--project-uuid="+self.arvrunner.project_uuid)
+ if self.enable_dev:
+ command.append("--enable-dev")
+
command.extend([workflowpath, "/var/lib/cwl/cwl.input.json"])
container_req["command"] = command
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index badf58938..9b9c2f161 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -365,7 +365,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
while keys:
page = keys[:pageSize]
- keys = keys[pageSize:]
try:
proc_states = table.list(filters=[["uuid", "in", page]]).execute(num_retries=self.num_retries)
except Exception:
@@ -381,6 +380,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
"new_attributes": p
}
})
+ keys = keys[pageSize:]
+
finish_poll = time.time()
remain_wait = self.poll_interval - (finish_poll - begin_poll)
except:
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 04fd1c443..ed6bcd008 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -131,7 +131,7 @@ def upload_dependencies(arvrunner, name, document_loader,
loadref_fields = set(("$import",))
scanobj = workflowobj
- if "id" in workflowobj:
+ if "id" in workflowobj and not workflowobj["id"].startswith("_:"):
# Need raw file content (before preprocessing) to ensure
# that external references in $include and $mixin are captured.
scanobj = loadref("", workflowobj["id"])
@@ -479,6 +479,7 @@ class Runner(Process):
self.intermediate_output_ttl = intermediate_output_ttl
self.priority = priority
self.secret_store = secret_store
+ self.enable_dev = loadingContext.enable_dev
self.submit_runner_cores = 1
self.submit_runner_ram = 1024 # defaut 1 GiB
commit c4e2ca60d9f572a7cbf194dcd4f15d26d8fa49b5
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Apr 26 14:53:20 2019 -0400
15028: Fix discover secondaryFiles after internal v1.1 update
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index c5fd7e3cf..badf58938 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -550,6 +550,10 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
if not runtimeContext.name:
runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
+ # Upload local file references in the job order.
+ job_order = upload_job_order(self, "%s input" % runtimeContext.name,
+ tool, job_order)
+
submitting = (runtimeContext.update_workflow or
runtimeContext.create_workflow or
(runtimeContext.submit and not
@@ -579,10 +583,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
loadingContext.metadata = tool.metadata
tool = load_tool(tool.tool, loadingContext)
- # Upload local file references in the job order.
- job_order = upload_job_order(self, "%s input" % runtimeContext.name,
- tool, job_order)
-
existing_uuid = runtimeContext.update_workflow
if existing_uuid or runtimeContext.create_workflow:
# Create a pipeline template or workflow record and exit.
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 3235f4763..04fd1c443 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -79,7 +79,7 @@ def find_defaults(d, op):
def setSecondary(t, fileobj, discovered):
if isinstance(fileobj, dict) and fileobj.get("class") == "File":
if "secondaryFiles" not in fileobj:
- fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf), "class": "File"} for sf in t["secondaryFiles"]])
+ fileobj["secondaryFiles"] = cmap([{"location": substitute(fileobj["location"], sf["pattern"]), "class": "File"} for sf in t["secondaryFiles"]])
if discovered is not None:
discovered[fileobj["location"]] = fileobj["secondaryFiles"]
elif isinstance(fileobj, list):
commit 675eb1e789edcb2b05e818943e9f8c333952f107
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Apr 26 14:08:41 2019 -0400
15028: Bump cwltool
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 37682bc9c..2b273017e 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20190425212529',
+ 'cwltool==1.0.20190426175817',
'schema-salad==4.1.20190305210046',
'typing >= 3.6.4',
'ruamel.yaml >=0.15.54, <=0.15.77',
commit 9e265731cf581ffb3650fd9db1aa0347650e2128
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Apr 26 14:07:05 2019 -0400
15028: Set the correct loading context to reload tools
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 768783df9..c5fd7e3cf 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -574,6 +574,9 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
# ArvadosCommandTool) because tool document may have been
# updated by upload_workflow_deps in ways that modify
# inheritance of hints or requirements.
+ loadingContext.loader = tool.doc_loader
+ loadingContext.avsc_names = tool.doc_schema
+ loadingContext.metadata = tool.metadata
tool = load_tool(tool.tool, loadingContext)
# Upload local file references in the job order.
@@ -647,8 +650,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
if runtimeContext.submit:
# Submit a runner job to run the workflow for us.
if self.work_api == "containers":
- loadingContext.loader = tool.doc_loader
- loadingContext.avsc_names = tool.doc_schema
if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and (not runtimeContext.always_submit_runner):
runtimeContext.runnerjob = tool.tool["id"]
else:
commit 3954f59491891999e117fe1b56d073d628775e5e
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Apr 25 17:42:49 2019 -0400
15028: Instead of suppressing update, reload old version for submit
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 95711762c..def720381 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -219,8 +219,11 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
def add_arv_hints():
cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*")
cwltool.command_line_tool.ACCEPTLIST_RE = cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE
- res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema.yml')
- use_custom_schema("v1.0", "http://arvados.org/cwl", res.read())
+ res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.0.yml')
+ res = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.1.yml')
+ customschema = res.read()
+ use_custom_schema("v1.0", "http://arvados.org/cwl", customschema)
+ use_custom_schema("v1.1.0-dev1", "http://arvados.org/cwl", customschema)
res.close()
cwltool.process.supportedProcessRequirements.extend([
"http://arvados.org/cwl#RunInSingleContainer",
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
similarity index 100%
copy from sdk/cwl/arvados_cwl/arv-cwl-schema.yml
copy to sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
similarity index 80%
rename from sdk/cwl/arvados_cwl/arv-cwl-schema.yml
rename to sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index dce1bd4d0..b9b9e6165 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -9,24 +9,6 @@ $namespaces:
$graph:
- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
-- name: cwltool:LoadListingRequirement
- type: record
- extends: cwl:ProcessRequirement
- inVocab: false
- fields:
- class:
- type: string
- doc: "Always 'LoadListingRequirement'"
- jsonldPredicate:
- "_id": "@type"
- "_type": "@vocab"
- loadListing:
- type:
- - "null"
- - type: enum
- name: LoadListingEnum
- symbols: [no_listing, shallow_listing, deep_listing]
-
- name: cwltool:Secrets
type: record
inVocab: false
@@ -47,28 +29,6 @@ $graph:
"_type": "@id"
refScope: 0
-- name: cwltool:TimeLimit
- type: record
- inVocab: false
- extends: cwl:ProcessRequirement
- doc: |
- Set an upper limit on the execution time of a CommandLineTool or
- ExpressionTool. A tool execution which exceeds the time limit may
- be preemptively terminated and considered failed. May also be
- used by batch systems to make scheduling decisions.
- fields:
- - name: class
- type: string
- doc: "Always 'TimeLimit'"
- jsonldPredicate:
- "_id": "@type"
- "_type": "@vocab"
- - name: timelimit
- type: [long, string]
- doc: |
- The time limit, in seconds. A time limit of zero means no
- time limit. Negative time limits are an error.
-
- name: RunInSingleContainer
type: record
extends: cwl:ProcessRequirement
@@ -196,22 +156,6 @@ $graph:
recommended minimum value for TTL is the expected duration of the
entire the workflow.
-- name: ReuseRequirement
- type: record
- extends: cwl:ProcessRequirement
- inVocab: false
- doc: |
- Enable/disable work reuse for current process. Default true (work reuse enabled).
- fields:
- - name: class
- type: string
- doc: "Always 'arv:ReuseRequirement'"
- jsonldPredicate:
- _id: "@type"
- _type: "@vocab"
- - name: enableReuse
- type: boolean
-
- name: WorkflowRunnerResources
type: record
extends: cwl:ProcessRequirement
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index ab2078e15..11efc0c1c 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -236,8 +236,11 @@ class ArvadosJob(JobBase):
try:
if record["state"] == "Complete":
processStatus = "success"
+ # we don't have the real exit code so fake it.
+ record["exit_code"] = 0
else:
processStatus = "permanentFail"
+ record["exit_code"] = 1
outputs = {}
try:
@@ -272,8 +275,8 @@ class ArvadosJob(JobBase):
outputs = done.done(self, record, dirs["tmpdir"],
dirs["outdir"], dirs["keep"])
except WorkflowException as e:
- # Only include a stack trace if in debug mode.
- # This is most likely a user workflow error and a stack trace may obfuscate more useful output.
+ # Only include a stack trace if in debug mode.
+ # This is most likely a user workflow error and a stack trace may obfuscate more useful output.
logger.error("%s unable to collect output from %s:\n%s",
self.arvrunner.label(self), record["output"], e, exc_info=(e if self.arvrunner.debug else False))
processStatus = "permanentFail"
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 20cff6060..768783df9 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -188,7 +188,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
self.loadingContext.fetcher_constructor = self.fetcher_constructor
self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
self.loadingContext.construct_tool_object = self.arv_make_tool
- self.loadingContext.do_update = False
# Add a custom logging handler to the root logger for runtime status reporting
# if running inside a container
@@ -551,21 +550,31 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
if not runtimeContext.name:
runtimeContext.name = self.name = tool.tool.get("label") or tool.metadata.get("label") or os.path.basename(tool.tool["id"])
- # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
- # Also uploads docker images.
- merged_map = upload_workflow_deps(self, tool)
+ submitting = (runtimeContext.update_workflow or
+ runtimeContext.create_workflow or
+ (runtimeContext.submit and not
+ (tool.tool["class"] == "CommandLineTool" and
+ runtimeContext.wait and
+ not runtimeContext.always_submit_runner)))
- # Reload tool object which may have been updated by
- # upload_workflow_deps
- # Don't validate this time because it will just print redundant errors.
loadingContext = self.loadingContext.copy()
- loadingContext.loader = tool.doc_loader
- loadingContext.avsc_names = tool.doc_schema
- loadingContext.metadata = tool.metadata
loadingContext.do_validate = False
+ loadingContext.do_update = False
+ if submitting:
+ # Document may have been auto-updated. Reload the original
+ # document with updating disabled because we want to
+ # submit the original document, not the auto-updated one.
+ tool = load_tool(tool.tool["id"], loadingContext)
+
+ # Upload direct dependencies of workflow steps, get back mapping of files to keep references.
+ # Also uploads docker images.
+ merged_map = upload_workflow_deps(self, tool)
- tool = self.arv_make_tool(tool.doc_loader.idx[tool.tool["id"]],
- loadingContext)
+ # Recreate process object (ArvadosWorkflow or
+ # ArvadosCommandTool) because tool document may have been
+ # updated by upload_workflow_deps in ways that modify
+ # inheritance of hints or requirements.
+ tool = load_tool(tool.tool, loadingContext)
# Upload local file references in the job order.
job_order = upload_job_order(self, "%s input" % runtimeContext.name,
@@ -638,6 +647,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
if runtimeContext.submit:
# Submit a runner job to run the workflow for us.
if self.work_api == "containers":
+ loadingContext.loader = tool.doc_loader
+ loadingContext.avsc_names = tool.doc_schema
if tool.tool["class"] == "CommandLineTool" and runtimeContext.wait and (not runtimeContext.always_submit_runner):
runtimeContext.runnerjob = tool.tool["id"]
else:
@@ -673,11 +684,6 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
"state": "RunningOnClient"}).execute(num_retries=self.num_retries)
logger.info("Pipeline instance %s", self.pipeline["uuid"])
- if not isinstance(tool, Runner):
- loadingContext.do_update = True
- tool = load_tool(tool.doc_loader.idx[tool.tool["id"]],
- loadingContext)
-
if runtimeContext.cwl_runner_job is not None:
self.uuid = runtimeContext.cwl_runner_job.get('uuid')
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 79c5fc209..37682bc9c 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -25,7 +25,7 @@ setup(name='arvados-cwl-runner',
download_url="https://github.com/curoverse/arvados.git",
license='Apache 2.0',
packages=find_packages(),
- package_data={'arvados_cwl': ['arv-cwl-schema.yml']},
+ package_data={'arvados_cwl': ['arv-cwl-schema-v1.0.yml', 'arv-cwl-schema-v1.1.yml']},
scripts=[
'bin/cwl-runner',
'bin/arvados-cwl-runner',
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20190423203253',
+ 'cwltool==1.0.20190425212529',
'schema-salad==4.1.20190305210046',
'typing >= 3.6.4',
'ruamel.yaml >=0.15.54, <=0.15.77',
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 2ddd06535..1f169cf23 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -513,7 +513,7 @@ class TestContainer(unittest.TestCase):
self.assertFalse(api.collections().create.called)
self.assertFalse(runner.runtime_status_error.called)
- arvjob.collect_outputs.assert_called_with("keep:abc+123")
+ arvjob.collect_outputs.assert_called_with("keep:abc+123", 0)
arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
commit 8a3e48d792f1b9f3ba12cefee4c0cdb50d85c847
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Apr 23 18:01:55 2019 -0400
15028: Fix reporting logs on container requests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index b194f3dfc..437640ac6 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -527,6 +527,7 @@ class RunnerContainer(Runner):
container = self.arvrunner.api.containers().get(
uuid=record["container_uuid"]
).execute(num_retries=self.arvrunner.num_retries)
+ container["log"] = record["log_uuid"]
except Exception:
logger.exception("%s while getting runner container", self.arvrunner.label(self))
self.arvrunner.output_callback({}, "permanentFail")
commit 09fcc3cf5ba690aa75c36dfef911c730f48f05ab
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Apr 23 17:32:33 2019 -0400
15028: Fixing 1.0 conformance tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index 2e4c45738..52df80f58 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -37,8 +37,12 @@ fi
cd "$WORKSPACE"
py=python
+pipcmd=pip
if [[ -n "$PYCMD" ]] ; then
- py="$PYCMD" ;
+ py="$PYCMD"
+ if [[ $py = python3 ]] ; then
+ pipcmd=pip3
+ fi
fi
(cd sdk/python && python setup.py sdist)
@@ -75,6 +79,6 @@ if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
fi
-docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
+docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
echo arv-keepdocker arvados/jobs $cwl_runner_version
arv-keepdocker arvados/jobs $cwl_runner_version
diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py
index 9b26ad706..c0e3e0de2 100644
--- a/sdk/cwl/arvados_cwl/done.py
+++ b/sdk/cwl/arvados_cwl/done.py
@@ -52,7 +52,7 @@ def done(self, record, tmpdir, outdir, keepdir):
def done_outputs(self, record, tmpdir, outdir, keepdir):
self.builder.outdir = outdir
self.builder.pathmapper.keepdir = keepdir
- return self.collect_outputs("keep:" + record["output"])
+ return self.collect_outputs("keep:" + record["output"], record["exit_code"])
crunchstat_re = re.compile(r"^\d{4}-\d\d-\d\d_\d\d:\d\d:\d\d [a-z0-9]{5}-8i9sb-[a-z0-9]{15} \d+ \d+ stderr crunchstat:")
timestamp_re = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+Z) (.*)")
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 2875dbc22..20cff6060 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -45,6 +45,7 @@ from ._version import __version__
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing, visit_class
from cwltool.command_line_tool import compute_checksums
+from cwltool.load_tool import load_tool
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
@@ -672,6 +673,11 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
"state": "RunningOnClient"}).execute(num_retries=self.num_retries)
logger.info("Pipeline instance %s", self.pipeline["uuid"])
+ if not isinstance(tool, Runner):
+ loadingContext.do_update = True
+ tool = load_tool(tool.doc_loader.idx[tool.tool["id"]],
+ loadingContext)
+
if runtimeContext.cwl_runner_job is not None:
self.uuid = runtimeContext.cwl_runner_job.get('uuid')
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 3ccf49f9e..79c5fc209 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,7 +33,7 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20190422193858',
+ 'cwltool==1.0.20190423203253',
'schema-salad==4.1.20190305210046',
'typing >= 3.6.4',
'ruamel.yaml >=0.15.54, <=0.15.77',
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index f9e370794..dd067e977 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -13,22 +13,19 @@
# (This dockerfile file must be located in the arvados/sdk/ directory because
# of the docker build root.)
-FROM debian:jessie
+FROM debian:9
MAINTAINER Ward Vandewege <ward at curoverse.com>
ENV DEBIAN_FRONTEND noninteractive
ARG pythoncmd=python
+ARG pipcmd=pip
RUN apt-get update -q && apt-get install -qy --no-install-recommends \
git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \
libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential
-RUN if [ "$pythoncmd" = "python3" ]; then \
- pip3 install -U setuptools six requests ; \
- else \
- pip install -U setuptools six requests ; \
- fi
+RUN $pipcmd install -U setuptools six requests
ARG sdk
ARG runner
@@ -40,10 +37,10 @@ ADD cwl/salad_dist/$salad /tmp/
ADD cwl/cwltool_dist/$cwltool /tmp/
ADD cwl/dist/$runner /tmp/
-RUN cd /tmp/arvados-python-client-* && $pythoncmd setup.py install
-RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pythoncmd setup.py install ; fi
-RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pythoncmd setup.py install ; fi
-RUN cd /tmp/arvados-cwl-runner-* && $pythoncmd setup.py install
+RUN cd /tmp/arvados-python-client-* && $pipcmd install .
+RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pipcmd install . ; fi
+RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install networkx==2.2 && $pipcmd install . ; fi
+RUN cd /tmp/arvados-cwl-runner-* && $pipcmd install .
# Install dependencies and set up system.
RUN /usr/sbin/adduser --disabled-password \
commit ac0dc6c57a2b4f736b6faf62421de56a3355db04
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Apr 5 13:12:45 2019 -0400
15028: Update cwltool/schema-salad deps, fix tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 8e6bff4f0..325fccb2c 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -288,6 +288,10 @@ class ArvadosWorkflow(Workflow):
adjustDirObjs(packed, keepmount)
self.wf_pdh = upload_workflow_collection(self.arvrunner, shortname(self.tool["id"]), packed)
+ self.loadingContext = self.loadingContext.copy()
+ self.loadingContext.metadata = self.loadingContext.metadata.copy()
+ self.loadingContext.metadata["http://commonwl.org/cwltool#original_cwlVersion"] = "v1.0"
+
wf_runner = cmap({
"class": "CommandLineTool",
"baseCommand": "cwltool",
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index eeb44dbd7..2875dbc22 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -187,6 +187,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
self.loadingContext.fetcher_constructor = self.fetcher_constructor
self.loadingContext.resolver = partial(collectionResolver, self.api, num_retries=self.num_retries)
self.loadingContext.construct_tool_object = self.arv_make_tool
+ self.loadingContext.do_update = False
# Add a custom logging handler to the root logger for runtime status reporting
# if running inside a container
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 9385bde63..3235f4763 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -31,6 +31,7 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
from cwltool.utils import aslist
from cwltool.builder import substitute
from cwltool.pack import pack
+from cwltool.update import INTERNAL_VERSION
import schema_salad.validate as validate
import arvados.collection
@@ -451,6 +452,10 @@ class Runner(Process):
collection_cache_size=256,
collection_cache_is_default=True):
+ loadingContext = loadingContext.copy()
+ loadingContext.metadata = loadingContext.metadata.copy()
+ loadingContext.metadata["cwlVersion"] = INTERNAL_VERSION
+
super(Runner, self).__init__(tool.tool, loadingContext)
self.arvrunner = runner
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 1052fb0d7..3ccf49f9e 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -33,8 +33,8 @@ setup(name='arvados-cwl-runner',
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20181217162649',
- 'schema-salad==3.0.20181129082112',
+ 'cwltool==1.0.20190422193858',
+ 'schema-salad==4.1.20190305210046',
'typing >= 3.6.4',
'ruamel.yaml >=0.15.54, <=0.15.77',
'arvados-python-client>=1.3.0.20190205182514',
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 07d962bf9..2ddd06535 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -57,7 +57,7 @@ class CollectionMock(object):
class TestContainer(unittest.TestCase):
def helper(self, runner, enable_reuse=True):
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -66,7 +66,7 @@ class TestContainer(unittest.TestCase):
"basedir": "",
"make_fs_access": make_fs_access,
"loader": Loader({}),
- "metadata": {"cwlVersion": "v1.0"}})
+ "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"basedir": "",
@@ -400,7 +400,7 @@ class TestContainer(unittest.TestCase):
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
tool = cmap({
"inputs": [],
@@ -607,7 +607,7 @@ class TestContainer(unittest.TestCase):
"portable_data_hash": "99999999999999999999999999999994+99",
"manifest_text": ". 99999999999999999999999999999994+99 0:0:file1 0:0:file2"}
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
tool = cmap({
"inputs": [
@@ -697,7 +697,7 @@ class TestContainer(unittest.TestCase):
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
tool = cmap({"arguments": ["md5sum", "example.conf"],
"class": "CommandLineTool",
@@ -803,7 +803,7 @@ class TestContainer(unittest.TestCase):
"class": "CommandLineTool",
"hints": [
{
- "class": "http://commonwl.org/cwltool#TimeLimit",
+ "class": "ToolTimeLimit",
"timelimit": 42
}
]
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 022d75be1..9cac68aa6 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -34,7 +34,7 @@ if not os.getenv('ARVADOS_DEBUG'):
class TestJob(unittest.TestCase):
def helper(self, runner, enable_reuse=True):
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -43,7 +43,7 @@ class TestJob(unittest.TestCase):
"basedir": "",
"make_fs_access": make_fs_access,
"loader": Loader({}),
- "metadata": {"cwlVersion": "v1.0"},
+ "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
"makeTool": runner.arv_make_tool})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "jobs",
@@ -343,7 +343,7 @@ class TestJob(unittest.TestCase):
class TestWorkflow(unittest.TestCase):
def helper(self, runner, enable_reuse=True):
- document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1.0-dev1")
make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -358,7 +358,7 @@ class TestWorkflow(unittest.TestCase):
"basedir": "",
"make_fs_access": make_fs_access,
"loader": document_loader,
- "metadata": {"cwlVersion": "v1.0"},
+ "metadata": {"cwlVersion": "v1.1.0-dev1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
"construct_tool_object": runner.arv_make_tool})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "jobs",
@@ -458,6 +458,7 @@ bytes(b'''{
@mock.patch("arvados.collection.Collection")
@mock.patch('arvados.commands.keepdocker.list_images_in_arv')
def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
+ # TODO copy this over to test_container
arvados_cwl.add_arv_hints()
api = mock.MagicMock()
@@ -475,16 +476,15 @@ bytes(b'''{
runner.num_retries = 0
loadingContext, runtimeContext = self.helper(runner)
-
+ loadingContext.do_update = True
tool, metadata = loadingContext.loader.resolve_ref("tests/wf/echo-wf.cwl")
- metadata["cwlVersion"] = tool["cwlVersion"]
mockcollection.side_effect = lambda *args, **kwargs: CollectionMock(mock.MagicMock(), *args, **kwargs)
arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
arvtool.formatgraph = None
it = arvtool.job({}, mock.MagicMock(), runtimeContext)
-
+
next(it).run(runtimeContext)
next(it).run(runtimeContext)
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 9535f6ba2..5f92cee94 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -84,6 +84,7 @@ def stubs(func):
stubs.api = mock.MagicMock()
stubs.api._rootDesc = get_rootDesc()
+ stubs.api._rootDesc["uuidPrefix"] = "zzzzz"
stubs.api.users().current().execute.return_value = {
"uuid": stubs.fake_user_uuid,
diff --git a/sdk/cwl/tests/wf/scatter2_subwf.cwl b/sdk/cwl/tests/wf/scatter2_subwf.cwl
index 2af115543..ecd36026f 100644
--- a/sdk/cwl/tests/wf/scatter2_subwf.cwl
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@ -6,7 +6,7 @@
"$graph": [
{
"class": "Workflow",
- "cwlVersion": "v1.0",
+ "cwlVersion": "v1.1.0-dev1",
"hints": [],
"id": "#main",
"inputs": [
@@ -59,10 +59,10 @@
"run": {
"baseCommand": "sleep",
"class": "CommandLineTool",
- "id": "#main/sleep1/subtool",
+ "id": "#main/sleep1/run/subtool",
"inputs": [
{
- "id": "#main/sleep1/subtool/sleeptime",
+ "id": "#main/sleep1/run/subtool/sleeptime",
"inputBinding": {
"position": 1
},
@@ -71,7 +71,7 @@
],
"outputs": [
{
- "id": "#main/sleep1/subtool/out",
+ "id": "#main/sleep1/run/subtool/out",
"outputBinding": {
"outputEval": "out"
},
@@ -83,5 +83,5 @@
]
}
],
- "cwlVersion": "v1.0"
+ "cwlVersion": "v1.1.0-dev1"
}
\ No newline at end of file
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list