[ARVADOS] updated: da42861619eb478cd1f01d58a1ebe59f1a25002e
Git user
git at public.curoverse.com
Mon Jan 30 18:39:36 EST 2017
Summary of changes:
build/build.list | 34 +--
build/package-build-dockerfiles/Makefile | 6 +-
.../{ubuntu1204 => ubuntu1604}/Dockerfile | 10 +-
.../{ubuntu1204 => ubuntu1604}/Dockerfile | 11 +-
.../ubuntu1604/etc-apt-preferences.d-arvados | 3 +
build/package-testing/deb-common-test-packages.sh | 2 +-
...ages-debian7.sh => test-packages-ubuntu1604.sh} | 0
build/run-build-packages-one-target.sh | 5 +-
build/run-build-packages.sh | 3 +
doc/_includes/_navbar_top.liquid | 2 +-
doc/api/methods.html.textile.liquid | 2 +
doc/sdk/cli/reference.html.textile.liquid | 13 +-
doc/user/index.html.textile.liquid | 2 +-
sdk/cli/bin/crunch-job | 9 +
sdk/cwl/arvados_cwl/__init__.py | 87 ++++--
sdk/cwl/arvados_cwl/arvcontainer.py | 38 ++-
sdk/cwl/arvados_cwl/arvdocker.py | 2 +
sdk/cwl/arvados_cwl/arvjob.py | 35 ++-
sdk/cwl/arvados_cwl/arvworkflow.py | 57 ++--
sdk/cwl/arvados_cwl/done.py | 35 ++-
sdk/cwl/arvados_cwl/fsaccess.py | 31 +-
sdk/cwl/arvados_cwl/pathmapper.py | 8 +-
sdk/cwl/arvados_cwl/runner.py | 180 ++++++++----
sdk/cwl/setup.py | 4 +-
sdk/cwl/tests/test_job.py | 17 +-
sdk/cwl/tests/test_submit.py | 245 ++++++++++------
sdk/cwl/tests/wf/expect_arvworkflow.cwl | 22 +-
sdk/cwl/tests/wf/expect_packed.cwl | 4 +-
sdk/cwl/tests/wf/scatter2_subwf.cwl | 116 +++++---
.../wf/{expect_packed.cwl => submit_wf_packed.cwl} | 8 +-
sdk/go/arvados/resource_list.go | 2 +
sdk/go/config/dump.go | 27 ++
sdk/go/keepclient/block_cache.go | 104 +++++++
sdk/go/keepclient/collectionreader.go | 321 ++++++++-------------
sdk/go/keepclient/collectionreader_test.go | 48 ++-
sdk/go/keepclient/keepclient.go | 9 +
.../api/app/controllers/application_controller.rb | 15 +-
.../controllers/arvados/v1/schema_controller.rb | 10 +-
.../api/config/initializers/noop_deep_munge.rb | 9 +
.../api/test/functional/arvados/v1/query_test.rb | 29 ++
services/api/test/integration/noop_deep_munge.rb | 35 +++
services/arv-git-httpd/main.go | 5 +
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 19 +-
services/crunch-run/crunchrun.go | 16 +-
services/crunch-run/crunchrun_test.go | 16 +-
services/keep-balance/balance_run_test.go | 4 +-
services/keep-balance/collection.go | 75 +++--
services/keep-balance/collection_test.go | 57 ++++
services/keep-balance/main.go | 19 +-
services/keep-web/handler.go | 52 +---
services/keep-web/handler_test.go | 43 ---
services/keep-web/main.go | 6 +
services/keep-web/ranges_test.go | 90 ++++++
services/keepproxy/keepproxy.go | 11 +-
services/keepstore/keepstore.go | 8 +-
55 files changed, 1320 insertions(+), 701 deletions(-)
copy build/package-build-dockerfiles/{ubuntu1204 => ubuntu1604}/Dockerfile (72%)
copy build/package-test-dockerfiles/{ubuntu1204 => ubuntu1604}/Dockerfile (55%)
create mode 100644 build/package-test-dockerfiles/ubuntu1604/etc-apt-preferences.d-arvados
rename build/package-testing/{test-packages-debian7.sh => test-packages-ubuntu1604.sh} (100%)
copy sdk/cwl/tests/wf/{expect_packed.cwl => submit_wf_packed.cwl} (53%)
create mode 100644 sdk/go/config/dump.go
create mode 100644 sdk/go/keepclient/block_cache.go
create mode 100644 services/api/config/initializers/noop_deep_munge.rb
create mode 100644 services/api/test/integration/noop_deep_munge.rb
create mode 100644 services/keep-balance/collection_test.go
create mode 100644 services/keep-web/ranges_test.go
via da42861619eb478cd1f01d58a1ebe59f1a25002e (commit)
via dc7d01f4d4031962ffd5734ca0c64146a7217e4a (commit)
via aa05b5d5ee2e42f2e456cca0c2f3cd835ab7cef4 (commit)
via fdecbc701188e64dc9e539da2dddbbeded709b42 (commit)
via 28aea3c6b1888c00a041992282c95fe595f85005 (commit)
via 436f5c768dbc97135490b6477efd1ff0482a9dda (commit)
via 71229e23918b698caa7c6c8b62b368d4aef2ab85 (commit)
via d6579d95bacbba5682d0c085f139681c93cb005a (commit)
via 827879be023e90d58eb681b3c930154739a0b27f (commit)
via 24b137a5b3313778e2db7f5d1e0c82daf0634a9c (commit)
via b001425779a0189b357c4e3a47734eda6b77ce2d (commit)
via 2b37f7a8e843638431dc9e8031c1c2c2a81c1e7e (commit)
via 1b8106da867aafbe6e4653485d5e644faee4c35d (commit)
via dcb7f718a7bf31c6e1227d28cdaac64fa769015b (commit)
via 95d8984418a5198639000f0a622da74af259b6c5 (commit)
via 8675eec8206c0e39753741864636c8a05ce02408 (commit)
via 799b5333c66ccd18fe1b0ee97c2a28be6e07b731 (commit)
via 9dd66f4d896785e0c6dce839ed1b411dd3c77bf0 (commit)
via 7dd717ca530fccae814c8c59a8c242d376fa0cef (commit)
via a7222f4df954c9ac973d58bb2f27a8f049dbbbc2 (commit)
via a3ca6693f522b59ea50d76cadde69b91c0a62cad (commit)
via 13c27ef91fc2516bdcdee94ba1ff2f08361dc51d (commit)
via 5ba9471d651c9f6235988d3ec680461ba4785453 (commit)
via 0d7ceb1e3498210d20859516de4ffdd59530b6d7 (commit)
via da0aee751d8cb039c9b6b85a03e7d62cb973e3b3 (commit)
via a20c77028c182b684a3acb791c7de4183319737e (commit)
via 1edd524ecd7b6776dd9b20ab81030e7c69c392b4 (commit)
via b172bbd9e4c780d9af79774aee81a1ac48b3a857 (commit)
via 8e569c16ba035b131c148441ca5a590fb49811ac (commit)
via 38ac8f3e065e74f6af172cee90144701d2f73b58 (commit)
via 1e7d93d8dcc2d896a71c38910f4fb6ef35408c0d (commit)
via 7fd60cc870863647127a438a085685d415c37a46 (commit)
via aea835bc965d42e225c2641b0210c4b521f6dc4e (commit)
via 16fe80b0e93ed8c8416b2dcbc0e2ad49bc850738 (commit)
via b8de9b3e62e82b806576b237be5f317bf378169f (commit)
via 39189b90bd2f73d4f4938049ffa4441a967ca24c (commit)
via dc5a33cbfc156c839515acb4fa6ea2f9162a0972 (commit)
via cd383b7168d9412f4f097438d590e919ff7a97d6 (commit)
via ddcaafee5dc10ee0104c108c9648f4d5024a83d9 (commit)
via 685af7fb2ae3a8ea162edd89eec61fdd4ca376f0 (commit)
via ae9f71231ed50eb35097c10c84b0070bcdcd22cf (commit)
via 01007e0a2c7cf5461ced83339f6abcfb6f9fac72 (commit)
via 6143bba5421756c78b282ee6c4da793d45a4523e (commit)
via 036c59ea6b19372e74f7ccacb5dcb2f522f99629 (commit)
via 7fa95f2db716ebfdb6312fa67b9b07bebb815b39 (commit)
via f40364c4d42e111b9da3873afcfaed2b49e7f182 (commit)
via 2f953026bc4baeccb78ca82acc4d07cad37625b8 (commit)
via 60d986b8908487c086eb4e402ac69669cb26108b (commit)
via c40389e0f064d4ea379d5f5471116936239a467a (commit)
via b6e15a581be2e5b2387fe18ecb01714fbc21d3aa (commit)
via 80f042bec0b25966498324cb6aafb7fc24a36e12 (commit)
via 1cdb5cbf83d34cb1a19c78478a7d5a28738e7bbf (commit)
via e8521124b0b8e647c58e99905100171ea5e30abd (commit)
via b07d8a573c74e7be56a2fd8c5064032c37d90c67 (commit)
via de4ecbc700759ff22e76948a58f7d70e5d3c1464 (commit)
via e1fd558686c78d6edfd460b7531ec9b559299889 (commit)
via 4696633ae5e955c267d50a14e790c93d833f41b0 (commit)
via a9ae80084e5b8897408a47fe081481cb913fd260 (commit)
via 609646134bcd8fc3a7fd500848220741ecc4a9d2 (commit)
via 4032e4dc4284391f21f21630bdb7ca88f8329d8e (commit)
via 0d0092442c009b8b3e387da18f828f3f0518b709 (commit)
via a4edc2a4a740d3908b907682a85e227eee80683b (commit)
via f65530a60d7b74da723d16eccd84576c53575401 (commit)
via be141b88fe871c7e72b19a5cf1e0aa56bf28283a (commit)
via 43dcf83e8b5332004bddd01be22476e68b1cef0f (commit)
via bb037e3872214d2b709964405775f476c6b3b550 (commit)
from a5e7c0c8bf829a41c4d02b48d5a9a40d6d225c8a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit da42861619eb478cd1f01d58a1ebe59f1a25002e
Author: radhika <radhika at curoverse.com>
Date: Mon Jan 30 16:20:24 2017 -0500
9397: if mount.Path ends with "/", trim it.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 13532ae..bd4c5cc 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -763,18 +763,22 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
bindSubdir = "." + bindSuffix[0:bindIdx]
bindFileName = bindSuffix[bindIdx+1:]
}
- pathIdx := strings.LastIndex(mnt.Path, "/")
+ mntPath := mnt.Path
+ if strings.HasSuffix(mntPath, "/") {
+ mntPath = mntPath[0 : len(mntPath)-1]
+ }
+ pathIdx := strings.LastIndex(mntPath, "/")
var pathSubdir, pathFileName string
if pathIdx >= 0 {
- pathSubdir = "." + mnt.Path[0:pathIdx]
- pathFileName = mnt.Path[pathIdx+1:]
+ pathSubdir = "." + mntPath[0:pathIdx]
+ pathFileName = mntPath[pathIdx+1:]
}
streams := strings.Split(collection.ManifestText, "\n")
for _, stream := range streams {
tokens := strings.Split(stream, " ")
- if tokens[0] == "."+mnt.Path {
+ if tokens[0] == "."+mntPath {
// path refers to this complete stream
- adjustedStream := strings.Replace(stream, "."+mnt.Path, "."+bindSuffix, -1)
+ adjustedStream := strings.Replace(stream, "."+mntPath, "."+bindSuffix, -1)
manifestText = adjustedStream + "\n"
break
} else {
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 76e052c..35ef013 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1219,7 +1219,7 @@ func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
"/tmp": {"kind": "tmp"},
"/tmp/foo": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367"},
"/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt"},
- "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1"},
+ "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/"},
"/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
"/tmp/foo/bar/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt"},
"stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
commit dc7d01f4d4031962ffd5734ca0c64146a7217e4a
Merge: a5e7c0c aa05b5d
Author: radhika <radhika at curoverse.com>
Date: Mon Jan 30 15:59:29 2017 -0500
Merge branch 'master' into 9397-prepopulate-output-directory
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list