[ARVADOS] updated: 777e716728a7da63ece00df7e5bb8be7f9a2a1a3
Git user
git at public.curoverse.com
Tue Apr 11 15:55:40 EDT 2017
Summary of changes:
.../app/controllers/collections_controller.rb | 14 +-
.../controllers/collections_controller_test.rb | 30 +++
build/build.list | 6 +-
.../package-test-dockerfiles/ubuntu1204/Dockerfile | 2 +-
build/rails-package-scripts/postinst.sh | 4 +-
build/run-tests.sh | 3 +
doc/_includes/_navbar_top.liquid | 2 +-
.../methods/container_requests.html.textile.liquid | 2 +
.../install-nodemanager.html.textile.liquid | 30 +++
sdk/cli/arvados-cli.gemspec | 2 +-
sdk/cli/test/test_arv-keep-get.rb | 4 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 3 +-
sdk/cwl/arvados_cwl/fsaccess.py | 5 +-
sdk/cwl/arvados_cwl/pathmapper.py | 11 +-
sdk/cwl/tests/arvados-tests.sh | 2 +-
sdk/cwl/tests/arvados-tests.yml | 38 +++
sdk/cwl/tests/cat.cwl | 8 +
sdk/cwl/tests/dir-job2.yml | 3 +
...dir-test-input.cwl => keep-dir-test-input2.cwl} | 3 +
...dir-test-input.cwl => keep-dir-test-input3.cwl} | 3 +
sdk/cwl/tests/octo.yml | 3 +
.../cwl/tests/octothorpe/item #1.txt | 0
sdk/cwl/tests/test_pathmapper.py | 22 +-
sdk/cwl/tests/wf/scatter2.cwl | 1 +
sdk/cwl/tests/wf/scatter2_subwf.cwl | 5 +-
sdk/go/keepclient/support.go | 10 +
sdk/python/arvados/__init__.py | 1 -
sdk/python/arvados/api.py | 10 +-
sdk/python/arvados/commands/get.py | 276 +++++++++++++++++++++
sdk/python/arvados/commands/ls.py | 57 ++++-
sdk/python/arvados/keep.py | 19 +-
sdk/python/bin/arv-get | 235 +-----------------
sdk/python/setup.py | 10 +-
sdk/python/tests/test_arv_get.py | 108 ++++++++
sdk/python/tests/test_arv_ls.py | 11 +-
sdk/python/tests/test_cache.py | 15 +-
sdk/ruby/arvados.gemspec | 2 +-
.../api/app/controllers/application_controller.rb | 46 +---
.../app/controllers/arvados/v1/nodes_controller.rb | 10 +-
services/api/app/models/arvados_model.rb | 104 +++++---
services/api/app/models/collection.rb | 40 ++-
services/api/app/models/container.rb | 93 ++++++-
services/api/app/models/container_request.rb | 234 +++++------------
services/api/app/models/node.rb | 20 +-
...0012505_add_output_ttl_to_container_requests.rb | 5 +
.../functional/arvados/v1/nodes_controller_test.rb | 10 +-
.../functional/arvados/v1/users_controller_test.rb | 24 +-
services/api/test/unit/collection_test.rb | 23 +-
services/api/test/unit/container_request_test.rb | 191 ++++++++------
services/api/test/unit/container_test.rb | 47 ++--
services/api/test/unit/node_test.rb | 12 +
services/crunch-run/crunchrun.go | 151 +++++++----
services/crunch-run/crunchrun_test.go | 161 ++++++------
services/fuse/arvados_fuse/command.py | 6 +-
services/fuse/arvados_fuse/unmount.py | 18 +-
services/fuse/tests/test_crunchstat.py | 13 +
services/fuse/tests/test_unmount.py | 40 ++-
.../arvnodeman/computenode/dispatch/__init__.py | 10 +-
.../arvnodeman/computenode/driver/__init__.py | 17 ++
.../arvnodeman/computenode/driver/gce.py | 12 +-
services/nodemanager/arvnodeman/config.py | 2 +
services/nodemanager/arvnodeman/daemon.py | 19 +-
services/nodemanager/arvnodeman/launcher.py | 3 +
services/nodemanager/arvnodeman/status.py | 65 +++++
services/nodemanager/doc/azure.example.cfg | 10 +
services/nodemanager/doc/ec2.example.cfg | 10 +
services/nodemanager/doc/gce.example.cfg | 10 +
services/nodemanager/doc/local.example.cfg | 4 +
services/nodemanager/setup.py | 22 +-
.../tests/test_computenode_driver_gce.py | 13 +-
services/nodemanager/tests/test_daemon.py | 8 +
services/nodemanager/tests/test_failure.py | 2 +-
services/nodemanager/tests/test_status.py | 55 ++++
73 files changed, 1623 insertions(+), 847 deletions(-)
create mode 100644 sdk/cwl/tests/cat.cwl
create mode 100644 sdk/cwl/tests/dir-job2.yml
copy sdk/cwl/tests/{keep-dir-test-input.cwl => keep-dir-test-input2.cwl} (77%)
copy sdk/cwl/tests/{keep-dir-test-input.cwl => keep-dir-test-input3.cwl} (76%)
create mode 100644 sdk/cwl/tests/octo.yml
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/octothorpe/item #1.txt (100%)
create mode 100755 sdk/python/arvados/commands/get.py
create mode 100644 sdk/python/tests/test_arv_get.py
create mode 100644 services/api/db/migrate/20170330012505_add_output_ttl_to_container_requests.rb
create mode 100644 services/fuse/tests/test_crunchstat.py
create mode 100644 services/nodemanager/arvnodeman/status.py
create mode 100644 services/nodemanager/tests/test_status.py
via 777e716728a7da63ece00df7e5bb8be7f9a2a1a3 (commit)
via 2c094e28cc4f19df4f6d0bc81355e7fc19e8c493 (commit)
via a779382603d2da2ec38ceb8a21262cc4f151f077 (commit)
via 26eb8f4cba5dd20ae49bf509a9a772ee4fe8b396 (commit)
via be2a34c57eab7cc184c08ec1059554cb97911276 (commit)
via 533bf8f1120c0f3fd74a31dfda990cc832b2aa51 (commit)
via 96b6e26749f3a7076b585758d44244777cc628c6 (commit)
via f08a738f1565b2dabde28dc9e8d1bad9622d5529 (commit)
via 466e0e2623bb85870943f48fe1ab3016dc32d652 (commit)
via 2f46fb5769f41ac9cddf8c46bd0d2ab094375e82 (commit)
via 909313b20ed5055d62113f0eea123f362869c3f4 (commit)
via 4f68551a87ff2cb88c23a2942abe2676a33f7cbb (commit)
via e7876a3ac520b128be7836e30172079ab2af5e45 (commit)
via f49bc2790a18de14e22e69fb39f93805488fdbb3 (commit)
via ab9a73d2c0b567d3c05d1d4d8463633a69eafda2 (commit)
via 86a6e4fe9347570090d1d1471f98675c3fb758f4 (commit)
via 157847615412907e87d24fb1767470d0157a4c30 (commit)
via 1b226fee3268f50ef670289fc5931a00866d69cd (commit)
via 4405570376295039b4c00577535a394eb011f8ad (commit)
via 2f52a6e7d9945f51d33b047466c4d927d494bb32 (commit)
via 0654fa160f872fe20ea4ada42a655f9d154c0833 (commit)
via 330f544dd257a43640eefc64b825b868f6e5be21 (commit)
via 351f06f68e8b079377bea195c67fed171b1e4b73 (commit)
via 75184884ed798b474e8b9e254045dc0f4354379e (commit)
via a869fa401696bc481380be239850ccf3edd808f6 (commit)
via 9dae3e6dde2993bc44e90539141b442c899cf114 (commit)
via 1c1c11c2c2d6cfd1d379cdbb166498514768be4e (commit)
via 1d0d28c791e88f817b1c01560ee8efd70bcab41e (commit)
via e3c48eaab3bbf538a03e575aab09cef59ee69f18 (commit)
via ed46350a4b4cd947d126b5e1e9a0514aa0b93532 (commit)
via 921d4321201be7542f9198202f75ba39c3f83d3f (commit)
via 1affdcd3cd34424a817ae350fd9ca236927fd538 (commit)
via 38fcd08dda022d0167840fbb65222fe99b75fcf5 (commit)
via 33ec0939cd5853d1c14a621c551ac2705efd1cd7 (commit)
via b594678083d6cd7efe425bb4be63a444f22ce153 (commit)
via 7b31dd40b31bdbf7473b50fd407cd6f636657c69 (commit)
via 7ce46899b5ed250aa909fa240b6be3e01637fb79 (commit)
via b50a3bcb38cf9e79416d8cc9b0d8b66249b3d473 (commit)
via 6da9f3666bc0ddee2d0be079cc30ba8b82804706 (commit)
via dfd3260e8b32ac4b9640e3e33bf8cf1e581bd917 (commit)
via 4bb024ae6903552c29613c851312f934dc6174b9 (commit)
via a03ce4056e503710caa1e95d315b92fb74c96abf (commit)
via 77f5a84ccc2b14438286ed05c6af183b8d8be605 (commit)
via 4a35e06b098bdd44a24fcaf77921aea5f371c84b (commit)
via 313415e33ca374898a371b9f22fc6546793f4688 (commit)
via 2c0c6e6ba25ec56bab8c224865a7979af577adaa (commit)
via 07c92074fe2b493dc8cd0abdad154e5b36d4adfc (commit)
via 1654f776b55bbb861f72c0dd3118ad1beec5db31 (commit)
via 988e9052575bc93db2909f3b9ef576f1043eddcb (commit)
via 3cc77494ac02e0f64a814cd4a898662fb7b329af (commit)
via ff3bb22d4b2bff5666907a6eeb6cd68cd3cbe22b (commit)
via b651984ff68c6f8add9b99427b205a2cb7c87462 (commit)
via 55c34fb97edfba5a014eb43b965130cf11dbda8a (commit)
via 28e59edcec4af09f78be965fa3abfaa9ee3f8dff (commit)
via 4af850d6898dea0c5f460f6acbc52f2e361cbf1d (commit)
via 736a629e16c388a5afecc97268bd7eadfd7ce3c2 (commit)
via 5ee2b9600ced7db8fefe141c837a894f33a1d129 (commit)
via c820bfc91be7635739bad0857ba3a385d1334b6a (commit)
via b9236fbe81426446e1b541a45e219bbe513fe8d0 (commit)
via fb1c8e81a200c11b1130f3a9af586f1bbf8c19b3 (commit)
via b351a877ff2d708acc5fe577d5ed5cc9b0469dde (commit)
via 0b04efefcc49b05df10a202cff5e423bbb587923 (commit)
via 65121f8db54a1ed15207d050e1f48c5fc26d646b (commit)
via 1a4a2f3219906220c0b4d7fd9b90325fa529408a (commit)
via 4437774e863465c0daa41dfd9716174e18d93122 (commit)
via a12ed889f9d0106ea26d0e2d6ff1f74e9ab14aac (commit)
via c507b0b072ad62c0087d059aedeaae8bae9b715f (commit)
via 0568c2d42703a7b839f2661968c05a23753f67c3 (commit)
via d6703f161c75a780ff645dc1ec980fca8a3e315b (commit)
via bebac87e773d73788de9273d65ca92db6878b2e7 (commit)
via e8e9262a0f5e1e5908d338ac1655a2cdfaecf23a (commit)
via 6fe136c2c7419dfb38a9af42898682701646b518 (commit)
via f34f20a9ac4b7ade7eeb05157a3b190d8a98c37e (commit)
via 154ae0d4b13329bae1033ab99095c00d0b0f66e2 (commit)
via ff8d14acd42952a21f5428e96d86e4a54c41be9a (commit)
via ae7f5a9c869927336dc81cf0552b955457a09647 (commit)
via 6828728001af20d8b75d841ead727c47d6ee2c96 (commit)
via 37313363a84fc41dcb87a8c9aa8b8502aaac256c (commit)
via de1e1bf2e6cd455135f754878966711db4dae9ec (commit)
via d040869ca4926c0eb39e072bdabf139bc1f25dd3 (commit)
via fbc867e0b02a0482d00000d76c5d0d343b7f252e (commit)
via c6a8839b1888d2eeb302ddfc675772428b8895a9 (commit)
via 78ca1460d58172f4555409298968583cd2ee70a1 (commit)
via 4afa501317cc04596377bcff8a7b2fccf7ab8d8d (commit)
from 0708d4a0b6db9426b9bb5addc1c592d526bd4ecd (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 777e716728a7da63ece00df7e5bb8be7f9a2a1a3
Merge: 0708d4a 2c094e2
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Apr 11 15:55:30 2017 -0400
Merge branch 'master' into 10401-limit-dir-expansion
diff --cc sdk/cwl/arvados_cwl/pathmapper.py
index be114eb,a8619a8..cddb408
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@@ -30,14 -31,12 +31,14 @@@ class ArvPathMapper(PathMapper)
def visit(self, srcobj, uploadfiles):
src = srcobj["location"]
- if srcobj["class"] == "File":
- if "#" in src:
- src = src[:src.index("#")]
- if isinstance(src, basestring) and ArvPathMapper.pdh_path.match(src):
- self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), "File")
- if src not in self._pathmap:
+ if "#" in src:
+ src = src[:src.index("#")]
+
+ if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
- self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], srcobj["class"], True)
++ self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True)
+
+ if src not in self._pathmap:
+ if src.startswith("file:"):
# Local FS ref, may need to be uploaded or may be on keep
# mount.
ab = abspath(src, self.input_basedir)
@@@ -48,21 -45,22 +49,21 @@@
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.fn[5:], "File", True)
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.unquote(st.fn[5:]), "File")
- elif src.startswith("_:"):
- if "contents" in srcobj:
- pass
- else:
- raise WorkflowException("File literal '%s' is missing contents" % src)
- elif src.startswith("arvwf:"):
- self._pathmap[src] = MapperEnt(src, src, "File")
++ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.unquote(st.fn[5:]), "File", True)
else:
raise WorkflowException("Input file path '%s' is invalid" % st)
- if "secondaryFiles" in srcobj:
- for l in srcobj["secondaryFiles"]:
- self.visit(l, uploadfiles)
- elif srcobj["class"] == "Directory":
- if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
- self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), "Directory")
+ elif src.startswith("_:"):
+ if srcobj["class"] == "File" and "contents" not in srcobj:
+ raise WorkflowException("File literal '%s' is missing `contents`" % src)
+ if srcobj["class"] == "Directory" and "listing" not in srcobj:
+ raise WorkflowException("Directory literal '%s' is missing `listing`" % src)
+ else:
+ self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
+
+ with SourceLine(srcobj, "secondaryFiles", WorkflowException):
+ for l in srcobj.get("secondaryFiles", []):
+ self.visit(l, uploadfiles)
+ with SourceLine(srcobj, "listing", WorkflowException):
for l in srcobj.get("listing", []):
self.visit(l, uploadfiles)
@@@ -93,7 -91,7 +94,7 @@@
loc = k["location"]
if loc in already_uploaded:
v = already_uploaded[loc]
- self._pathmap[loc] = MapperEnt(v.resolved, self.collection_pattern % v.resolved[5:], "File", True)
- self._pathmap[loc] = MapperEnt(v.resolved, self.collection_pattern % urllib.unquote(v.resolved[5:]), "File")
++ self._pathmap[loc] = MapperEnt(v.resolved, self.collection_pattern % urllib.unquote(v.resolved[5:]), "File", True)
for srcobj in referenced_files:
self.visit(srcobj, uploadfiles)
@@@ -108,8 -106,7 +109,8 @@@
project=self.arvrunner.project_uuid)
for src, ab, st in uploadfiles:
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % st.keepref,
- self._pathmap[src] = MapperEnt(urllib.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:], "File")
++ self._pathmap[src] = MapperEnt(urllib.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:],
+ "Directory" if os.path.isdir(ab) else "File", True)
self.arvrunner.add_uploaded(src, self._pathmap[src])
for srcobj in referenced_files:
diff --cc sdk/cwl/tests/test_pathmapper.py
index a81a853,3b6af04..b39a984
--- a/sdk/cwl/tests/test_pathmapper.py
+++ b/sdk/cwl/tests/test_pathmapper.py
@@@ -41,19 -40,19 +41,25 @@@ class TestPathmap(unittest.TestCase)
p._pathmap)
@mock.patch("arvados.commands.run.uploadfiles")
-- def test_upload(self, upl):
++ @mock.patch("arvados.commands.run.statfile")
++ def test_upload(self, statfile, upl):
"""Test pathmapper uploading files."""
arvrunner = arvados_cwl.ArvCwlRunner(self.api)
++ def statfile_mock(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
++ st = arvados.commands.run.UploadFile("", "tests/hw.py")
++ return st
++
upl.side_effect = upload_mock
++ statfile.side_effect = statfile_mock
p = ArvPathMapper(arvrunner, [{
"class": "File",
-- "location": "tests/hw.py"
++ "location": "file:tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
++ self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
@mock.patch("arvados.commands.run.uploadfiles")
@@@ -61,16 -60,16 +67,16 @@@
"""Test pathmapper handling previously uploaded files."""
arvrunner = arvados_cwl.ArvCwlRunner(self.api)
- arvrunner.add_uploaded('tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File', staged=True))
- arvrunner.add_uploaded('tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File'))
++ arvrunner.add_uploaded('file:tests/hw.py', MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='', type='File', staged=True))
upl.side_effect = upload_mock
p = ArvPathMapper(arvrunner, [{
"class": "File",
-- "location": "tests/hw.py"
++ "location": "file:tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
++ self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
@mock.patch("arvados.commands.run.uploadfiles")
@@@ -90,8 -89,8 +96,8 @@@
p = ArvPathMapper(arvrunner, [{
"class": "File",
-- "location": "tests/hw.py"
++ "location": "file:tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
- self.assertEqual({'tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File')},
++ self.assertEqual({'file:tests/hw.py': MapperEnt(resolved='keep:99999999999999999999999999999991+99/hw.py', target='/test/99999999999999999999999999999991+99/hw.py', type='File', staged=True)},
p._pathmap)
diff --cc sdk/cwl/tests/wf/scatter2.cwl
index f73ec2b,f73ec2b..83b3537
--- a/sdk/cwl/tests/wf/scatter2.cwl
+++ b/sdk/cwl/tests/wf/scatter2.cwl
@@@ -44,6 -44,6 +44,7 @@@ steps
out: [out]
run:
class: CommandLineTool
++ id: subtool
inputs:
sleeptime:
type: int
diff --cc sdk/cwl/tests/wf/scatter2_subwf.cwl
index df4d992,df4d992..dd4c705
--- a/sdk/cwl/tests/wf/scatter2_subwf.cwl
+++ b/sdk/cwl/tests/wf/scatter2_subwf.cwl
@@@ -50,9 -50,9 +50,10 @@@
"run": {
"baseCommand": "sleep",
"class": "CommandLineTool",
++ "id": "#main/sleep1/subtool",
"inputs": [
{
-- "id": "#main/sleep1/sleeptime",
++ "id": "#main/sleep1/subtool/sleeptime",
"inputBinding": {
"position": 1
},
@@@ -61,7 -61,7 +62,7 @@@
],
"outputs": [
{
-- "id": "#main/sleep1/out",
++ "id": "#main/sleep1/subtool/out",
"outputBinding": {
"outputEval": "out"
},
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list