[ARVADOS] updated: 1.3.0-1456-g9a02d4117
Git user
git at public.curoverse.com
Tue Aug 6 19:45:24 UTC 2019
Summary of changes:
doc/user/cwl/cwl-run-options.html.textile.liquid | 26 ++++++++++++------------
doc/user/cwl/cwl-versions.html.textile.liquid | 14 -------------
sdk/cwl/arvados_cwl/__init__.py | 12 +++++------
sdk/cwl/arvados_cwl/executor.py | 7 -------
sdk/cwl/tests/wf/runin-reqs-wf.cwl | 4 ++--
sdk/cwl/tests/wf/runin-reqs-wf3.cwl | 4 ++--
sdk/cwl/tests/wf/runin-reqs-wf4.cwl | 4 ++--
7 files changed, 25 insertions(+), 46 deletions(-)
via 9a02d4117bfc1372ce5b6baedc9303f2500b3197 (commit)
via bdb3e297e65ed39f66b33a6fb4c3b3b0e17e6fd5 (commit)
from 40d4eb5c5a942d82d158ab9d4d3b3fb1b44fde83 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9a02d4117bfc1372ce5b6baedc9303f2500b3197
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Aug 6 15:44:43 2019 -0400
15181: Tweak a few more integration tests to request fewer resources
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/wf/runin-reqs-wf.cwl b/sdk/cwl/tests/wf/runin-reqs-wf.cwl
index 9d23c6a87..e8bbc08d8 100644
--- a/sdk/cwl/tests/wf/runin-reqs-wf.cwl
+++ b/sdk/cwl/tests/wf/runin-reqs-wf.cwl
@@ -34,7 +34,7 @@ steps:
hints:
- class: arv:RunInSingleContainer
- class: ResourceRequirement
- ramMin: $(inputs.count*128)
+ ramMin: $(inputs.count*32)
- class: arv:APIRequirement
scatter: count
run:
@@ -58,4 +58,4 @@ steps:
type: int
script: File
outputs: []
- arguments: [python, $(inputs.script), $(inputs.count * 128)]
+ arguments: [python, $(inputs.script), $(inputs.count * 32)]
diff --git a/sdk/cwl/tests/wf/runin-reqs-wf3.cwl b/sdk/cwl/tests/wf/runin-reqs-wf3.cwl
index 3accb324f..e07ff8def 100644
--- a/sdk/cwl/tests/wf/runin-reqs-wf3.cwl
+++ b/sdk/cwl/tests/wf/runin-reqs-wf3.cwl
@@ -50,10 +50,10 @@ steps:
id: subtool
hints:
- class: ResourceRequirement
- ramMin: $(inputs.count*128)
+ ramMin: $(inputs.count*32)
inputs:
count:
type: int
script: File
outputs: []
- arguments: [python, $(inputs.script), $(inputs.count * 128)]
+ arguments: [python, $(inputs.script), $(inputs.count * 32)]
diff --git a/sdk/cwl/tests/wf/runin-reqs-wf4.cwl b/sdk/cwl/tests/wf/runin-reqs-wf4.cwl
index 9a26d0113..05e73c8df 100644
--- a/sdk/cwl/tests/wf/runin-reqs-wf4.cwl
+++ b/sdk/cwl/tests/wf/runin-reqs-wf4.cwl
@@ -53,10 +53,10 @@ steps:
id: subtool
hints:
- class: ResourceRequirement
- ramMin: 128
+ ramMin: 32
inputs:
count:
type: int
script: File
outputs: []
- arguments: [python, $(inputs.script), "128"]
+ arguments: [python, $(inputs.script), "32"]
commit bdb3e297e65ed39f66b33a6fb4c3b3b0e17e6fd5
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Aug 6 15:44:27 2019 -0400
15181: Remove some more jobs API references
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/doc/user/cwl/cwl-run-options.html.textile.liquid b/doc/user/cwl/cwl-run-options.html.textile.liquid
index 27970f440..725528f44 100644
--- a/doc/user/cwl/cwl-run-options.html.textile.liquid
+++ b/doc/user/cwl/cwl-run-options.html.textile.liquid
@@ -32,25 +32,25 @@ table(table table-bordered table-condensed).
|==--debug==| Print even more logging|
|==--metrics==| Print timing metrics|
|==--tool-help==| Print command line help for tool|
-|==--enable-reuse==| Enable job or container reuse (default)|
-|==--disable-reuse==| Disable job or container reuse|
-|==--project-uuid UUID==| Project that will own the workflow jobs, if not provided, will go to home project.|
+|==--enable-reuse==| Enable container reuse (default)|
+|==--disable-reuse==| Disable container reuse|
+|==--project-uuid UUID==| Project that will own the workflow containers, if not provided, will go to home project.|
|==--output-name OUTPUT_NAME==|Name to use for collection that stores the final output.|
|==--output-tags OUTPUT_TAGS==|Tags for the final output collection separated by commas, e.g., =='--output-tags tag0,tag1,tag2'==.|
-|==--ignore-docker-for-reuse==|Ignore Docker image version when deciding whether to reuse past jobs.|
+|==--ignore-docker-for-reuse==|Ignore Docker image version when deciding whether to reuse past containers.|
|==--submit==| Submit workflow runner to Arvados to manage the workflow (default).|
-|==--local==| Run workflow on local host (still submits jobs to Arvados).|
+|==--local==| Run workflow on local host (still submits containers to Arvados).|
|==--create-template==| (Deprecated) synonym for --create-workflow.|
-|==--create-workflow==| Create an Arvados workflow (if using the 'containers' API) or pipeline template (if using the 'jobs' API). See --api.|
+|==--create-workflow==| Register an Arvados workflow that can be run from Workbench|
|==--update-workflow== UUID|Update an existing Arvados workflow or pipeline template with the given UUID.|
-|==--wait==| After submitting workflow runner job, wait for completion.|
-|==--no-wait==| Submit workflow runner job and exit.|
+|==--wait==| After submitting workflow runner, wait for completion.|
+|==--no-wait==| Submit workflow runner and exit.|
|==--log-timestamps==| Prefix logging lines with timestamp|
|==--no-log-timestamps==| No timestamp on logging lines|
-|==--api== {jobs,containers}|Select work submission API. Default is 'jobs' if that API is available, otherwise 'containers'.|
+|==--api== {containers}|Select work submission API. Only supports 'containers'|
|==--compute-checksum==| Compute checksum of contents while collecting outputs|
-|==--submit-runner-ram== SUBMIT_RUNNER_RAM|RAM (in MiB) required for the workflow runner job (default 1024)|
-|==--submit-runner-image== SUBMIT_RUNNER_IMAGE|Docker image for workflow runner job|
+|==--submit-runner-ram== SUBMIT_RUNNER_RAM|RAM (in MiB) required for the workflow runner (default 1024)|
+|==--submit-runner-image== SUBMIT_RUNNER_IMAGE|Docker image for workflow runner|
|==--always-submit-runner==|When invoked with --submit --wait, always submit a runner to manage the workflow, even when only running a single CommandLineTool|
|==--submit-request-uuid== UUID|Update and commit to supplied container request instead of creating a new one (containers API only).|
|==--submit-runner-cluster== CLUSTER_ID|Submit workflow runner to a remote cluster (containers API only)|
@@ -60,7 +60,7 @@ table(table table-bordered table-condensed).
|==--storage-classes== STORAGE_CLASSES|Specify comma separated list of storage classes to be used when saving workflow output to Keep.|
|==--intermediate-output-ttl== N|If N > 0, intermediate output collections will be trashed N seconds after creation. Default is 0 (don't trash).|
|==--priority== PRIORITY|Workflow priority (range 1..1000, higher has precedence over lower, containers api only)|
-|==--thread-count== THREAD_COUNT|Number of threads to use for job submit and output collection.|
+|==--thread-count== THREAD_COUNT|Number of threads to use for container submit and output collection.|
|==--http-timeout== HTTP_TIMEOUT|API request timeout in seconds. Default is 300 seconds (5 minutes).|
|==--trash-intermediate==|Immediately trash intermediate outputs on workflow success.|
|==--no-trash-intermediate==|Do not trash intermediate outputs (default).|
@@ -106,7 +106,7 @@ qr1hi-8i9sb-fm2n3b1w0l6bskg
h3(#local). Control a workflow locally
-To run a workflow with local control, use @--local at . This means that the host where you run @arvados-cwl-runner@ will be responsible for submitting jobs, however, the jobs themselves will still run on the Arvados cluster. With @--local@, if you interrupt @arvados-cwl-runner@ or log out, the workflow will be terminated.
+To run a workflow with local control, use @--local at . This means that the host where you run @arvados-cwl-runner@ will be responsible for submitting containers, however, the containers themselves will still run on the Arvados cluster. With @--local@, if you interrupt @arvados-cwl-runner@ or log out, the workflow will be terminated.
<notextile>
<pre><code>~/arvados/doc/user/cwl/bwa-mem$ <span class="userinput">arvados-cwl-runner --local bwa-mem.cwl bwa-mem-input.yml</span>
diff --git a/doc/user/cwl/cwl-versions.html.textile.liquid b/doc/user/cwl/cwl-versions.html.textile.liquid
index 93b2e5907..7bb9fdcbe 100644
--- a/doc/user/cwl/cwl-versions.html.textile.liquid
+++ b/doc/user/cwl/cwl-versions.html.textile.liquid
@@ -18,17 +18,3 @@ If a step requires network access, use "NetworkAccess":https://www.commonwl.org/
To prevent misbehaving steps from running forever and wasting resources, you can fail the step if it exceeds a certain running time with "ToolTimeLimit":https://www.commonwl.org/v1.1/CommandLineTool.html#ToolTimeLimit instead of the deprecated @cwltool:TimeLimit@ .
To control if an individual step can be reused, use "WorkReuse":https://www.commonwl.org/v1.1/CommandLineTool.html#WorkReuse instead of the deprecated @arv:ReuseRequirement at .
-
-h2(#migrate). Differences in running CWL on the legacy jobs API vs containers API
-
-Most users can ignore this section.
-
-When migrating your Arvados cluster from using the jobs API (--api=jobs) (sometimes referred to as "crunch v1") to the containers API (--api=containers) ("crunch v2") there are a few differences in behavior:
-
-A tool may fail to find an input file that could be found when run under the jobs API. This is because tools are limited to accessing collections explicitly listed in the input, and further limited to those individual files or subdirectories that are listed. For example, given an explicit file input @/dir/subdir/file1.txt@, a tool will not be allowed to implicitly access a file in the parent directory @/dir/file2.txt at . Use @secondaryFiles@ or a @Directory@ for files that need to be grouped together.
-
-A tool may fail when attempting to rename or delete a file in the output directory. This may happen because files listed in @InitialWorkDirRequirement@ appear in the output directory as normal files (not symlinks) but cannot be moved, renamed or deleted unless marked as "writable" in CWL. These files will be added to the output collection but without any additional copies of the underlying data.
-
-A tool may fail when attempting to access the network. This may happen because, unlike the jobs API, under the containers API network access is disabled by default. Tools which require network access should add "arv:APIRequirement: {}":cwl-extensions.html#APIRequirement to the @requirements@ section.
-
-CWL v1.1 is not supported by the Jobs API.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 088609fd7..4c9838580 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -96,10 +96,10 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--enable-reuse", action="store_true",
default=True, dest="enable_reuse",
- help="Enable job or container reuse (default)")
+ help="Enable container reuse (default)")
exgroup.add_argument("--disable-reuse", action="store_false",
default=True, dest="enable_reuse",
- help="Disable job or container reuse")
+ help="Disable container reuse")
parser.add_argument("--project-uuid", metavar="UUID", help="Project that will own the workflow containers, if not provided, will go to home project.")
parser.add_argument("--output-name", help="Name to use for collection that stores the final output.", default=None)
@@ -154,10 +154,10 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--submit-request-uuid",
default=None,
- help="Update and commit to supplied container request instead of creating a new one (containers API only).",
+ help="Update and commit to supplied container request instead of creating a new one.",
metavar="UUID")
exgroup.add_argument("--submit-runner-cluster",
- help="Submit workflow runner to a remote cluster (containers API only)",
+ help="Submit workflow runner to a remote cluster",
default=None,
metavar="CLUSTER_ID")
@@ -185,7 +185,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
default=0)
parser.add_argument("--priority", type=int,
- help="Workflow priority (range 1..1000, higher has precedence over lower, containers api only)",
+ help="Workflow priority (range 1..1000, higher has precedence over lower)",
default=DEFAULT_PRIORITY)
parser.add_argument("--disable-validate", dest="do_validate",
@@ -297,7 +297,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
return 1
# Note that unless in debug mode, some stack traces related to user
- # workflow errors may be suppressed. See ArvadosJob.done().
+ # workflow errors may be suppressed.
if arvargs.debug:
logger.setLevel(logging.DEBUG)
logging.getLogger('arvados').setLevel(logging.DEBUG)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 97e6b736a..eed2fe19d 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -399,18 +399,11 @@ The 'jobs' API is no longer supported.
def check_features(self, obj, parentfield=""):
if isinstance(obj, dict):
- if obj.get("writable") and self.work_api != "containers":
- raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported with --api=jobs")
if obj.get("class") == "DockerRequirement":
if obj.get("dockerOutputDirectory"):
- if self.work_api != "containers":
- raise SourceLine(obj, "dockerOutputDirectory", UnsupportedRequirement).makeError(
- "Option 'dockerOutputDirectory' of DockerRequirement not supported with --api=jobs.")
if not obj.get("dockerOutputDirectory").startswith('/'):
raise SourceLine(obj, "dockerOutputDirectory", validate.ValidationException).makeError(
"Option 'dockerOutputDirectory' must be an absolute path.")
- if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers":
- raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs")
if obj.get("class") == "InplaceUpdateRequirement":
if obj["inplaceUpdate"] and parentfield == "requirements":
raise SourceLine(obj, "class", UnsupportedRequirement).makeError("InplaceUpdateRequirement not supported for keep collections.")
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list