[ARVADOS] updated: 54f4d656aac507205396969dd16cb01570a9f707
Git user
git at public.curoverse.com
Wed Aug 10 15:04:42 EDT 2016
Summary of changes:
crunch_scripts/cwl-runner | 8 ++++++--
sdk/cwl/arvados_cwl/__init__.py | 4 +++-
sdk/cwl/arvados_cwl/pathmapper.py | 1 -
sdk/cwl/tests/arvados-tests.sh | 5 +++++
sdk/cwl/tests/arvados-tests.yml | 10 ++++++++++
sdk/cwl/tests/dir-job.yml | 3 +++
sdk/cwl/tests/dir6.cwl | 21 +++++++++++++++++++++
sdk/cwl/tests/runner.sh | 2 ++
.../tests/__init__.py => sdk/cwl/tests/testdir/a | 0
.../tests/__init__.py => sdk/cwl/tests/testdir/b | 0
.../tests/__init__.py => sdk/cwl/tests/testdir/c/d | 0
11 files changed, 50 insertions(+), 4 deletions(-)
create mode 100755 sdk/cwl/tests/arvados-tests.sh
create mode 100644 sdk/cwl/tests/arvados-tests.yml
create mode 100644 sdk/cwl/tests/dir-job.yml
create mode 100644 sdk/cwl/tests/dir6.cwl
create mode 100755 sdk/cwl/tests/runner.sh
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/testdir/a (100%)
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/testdir/b (100%)
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/testdir/c/d (100%)
via 54f4d656aac507205396969dd16cb01570a9f707 (commit)
from 60cd3e84fb2067191bc7f26490f301e2bbc07a4a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 54f4d656aac507205396969dd16cb01570a9f707
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Aug 10 13:45:56 2016 -0400
9751: Fix direct to keep directory references.
Add arvados-specific integration test for arvados-cwl-runner.
diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 2a1873a..fe4e800 100755
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -18,8 +18,9 @@ import logging
import os
import json
import argparse
+import re
from arvados.api import OrderedJsonModel
-from cwltool.process import adjustFileObjs
+from cwltool.process import adjustFileObjs, adjustDirObjs
from cwltool.load_tool import load_tool
# Print package versions
@@ -30,8 +31,10 @@ api = arvados.api("v1")
try:
job_order_object = arvados.current_job()['script_parameters']
+ pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$')
+
def keeppath(v):
- if arvados.util.keep_locator_pattern.match(v):
+ if pdh_path.match(v):
return "keep:%s" % v
else:
return v
@@ -49,6 +52,7 @@ try:
}
adjustFileObjs(job_order_object, keeppathObj)
+ adjustDirObjs(job_order_object, keeppathObj)
runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7bd9df3..fd9e74f 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -124,7 +124,9 @@ class ArvCwlRunner(object):
self.debug = kwargs.get("debug")
self.ignore_docker_for_reuse = kwargs.get("ignore_docker_for_reuse")
- self.fs_access = kwargs["make_fs_access"](kwargs["basedir"])
+ make_fs_access = kwargs.get("make_fs_access") or partial(CollectionFsAccess, api_client=self.api)
+ self.fs_access = make_fs_access(kwargs["basedir"])
+ kwargs["make_fs_access"] = make_fs_access
kwargs["enable_reuse"] = kwargs.get("enable_reuse")
kwargs["use_container"] = True
kwargs["tmpdir_prefix"] = "tmp"
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 24c319c..7e9a159 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -52,7 +52,6 @@ class ArvPathMapper(PathMapper):
elif srcobj["class"] == "Directory":
if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "Directory")
- else:
for l in srcobj["listing"]:
self.visit(l, uploadfiles)
diff --git a/sdk/cwl/tests/arvados-tests.sh b/sdk/cwl/tests/arvados-tests.sh
new file mode 100755
index 0000000..8646704
--- /dev/null
+++ b/sdk/cwl/tests/arvados-tests.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+if ! arv-get d7514270f356df848477718d58308cc4+94 > /dev/null ; then
+ arv-put --portable-data-hash testdir
+fi
+exec cwltest --test arvados-tests.yml --tool $PWD/runner.sh
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
new file mode 100644
index 0000000..2526258
--- /dev/null
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -0,0 +1,10 @@
+- job: dir-job.yml
+ output:
+ "outlist": {
+ "size": 20,
+ "location": "output.txt",
+ "class": "File",
+ "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592"
+ }
+ tool: dir6.cwl
+ doc: Test directory in keep
diff --git a/sdk/cwl/tests/dir-job.yml b/sdk/cwl/tests/dir-job.yml
new file mode 100644
index 0000000..91204d7
--- /dev/null
+++ b/sdk/cwl/tests/dir-job.yml
@@ -0,0 +1,3 @@
+indir:
+ class: Directory
+ location: keep:d7514270f356df848477718d58308cc4+94
\ No newline at end of file
diff --git a/sdk/cwl/tests/dir6.cwl b/sdk/cwl/tests/dir6.cwl
new file mode 100644
index 0000000..93362b5
--- /dev/null
+++ b/sdk/cwl/tests/dir6.cwl
@@ -0,0 +1,21 @@
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+ - class: ShellCommandRequirement
+inputs:
+ indir:
+ type: Directory
+ inputBinding:
+ prefix: cd
+ position: -1
+outputs:
+ outlist:
+ type: File
+ outputBinding:
+ glob: output.txt
+arguments: [
+ {shellQuote: false, valueFrom: "&&"},
+ "find", ".",
+ {shellQuote: false, valueFrom: "|"},
+ "sort"]
+stdout: output.txt
\ No newline at end of file
diff --git a/sdk/cwl/tests/runner.sh b/sdk/cwl/tests/runner.sh
new file mode 100755
index 0000000..22ede5c
--- /dev/null
+++ b/sdk/cwl/tests/runner.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec arvados-cwl-runner --disable-reuse --compute-checksum "$@"
diff --git a/sdk/cwl/tests/testdir/a b/sdk/cwl/tests/testdir/a
new file mode 100644
index 0000000..e69de29
diff --git a/sdk/cwl/tests/testdir/b b/sdk/cwl/tests/testdir/b
new file mode 100644
index 0000000..e69de29
diff --git a/sdk/cwl/tests/testdir/c/d b/sdk/cwl/tests/testdir/c/d
new file mode 100644
index 0000000..e69de29
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list