[ARVADOS] updated: da42861619eb478cd1f01d58a1ebe59f1a25002e

Git user git at public.curoverse.com
Mon Jan 30 18:39:36 EST 2017


Summary of changes:
 build/build.list                                   |  34 +--
 build/package-build-dockerfiles/Makefile           |   6 +-
 .../{ubuntu1204 => ubuntu1604}/Dockerfile          |  10 +-
 .../{ubuntu1204 => ubuntu1604}/Dockerfile          |  11 +-
 .../ubuntu1604/etc-apt-preferences.d-arvados       |   3 +
 build/package-testing/deb-common-test-packages.sh  |   2 +-
 ...ages-debian7.sh => test-packages-ubuntu1604.sh} |   0
 build/run-build-packages-one-target.sh             |   5 +-
 build/run-build-packages.sh                        |   3 +
 doc/_includes/_navbar_top.liquid                   |   2 +-
 doc/api/methods.html.textile.liquid                |   2 +
 doc/sdk/cli/reference.html.textile.liquid          |  13 +-
 doc/user/index.html.textile.liquid                 |   2 +-
 sdk/cli/bin/crunch-job                             |   9 +
 sdk/cwl/arvados_cwl/__init__.py                    |  87 ++++--
 sdk/cwl/arvados_cwl/arvcontainer.py                |  38 ++-
 sdk/cwl/arvados_cwl/arvdocker.py                   |   2 +
 sdk/cwl/arvados_cwl/arvjob.py                      |  35 ++-
 sdk/cwl/arvados_cwl/arvworkflow.py                 |  57 ++--
 sdk/cwl/arvados_cwl/done.py                        |  35 ++-
 sdk/cwl/arvados_cwl/fsaccess.py                    |  31 +-
 sdk/cwl/arvados_cwl/pathmapper.py                  |   8 +-
 sdk/cwl/arvados_cwl/runner.py                      | 180 ++++++++----
 sdk/cwl/setup.py                                   |   4 +-
 sdk/cwl/tests/test_job.py                          |  17 +-
 sdk/cwl/tests/test_submit.py                       | 245 ++++++++++------
 sdk/cwl/tests/wf/expect_arvworkflow.cwl            |  22 +-
 sdk/cwl/tests/wf/expect_packed.cwl                 |   4 +-
 sdk/cwl/tests/wf/scatter2_subwf.cwl                | 116 +++++---
 .../wf/{expect_packed.cwl => submit_wf_packed.cwl} |   8 +-
 sdk/go/arvados/resource_list.go                    |   2 +
 sdk/go/config/dump.go                              |  27 ++
 sdk/go/keepclient/block_cache.go                   | 104 +++++++
 sdk/go/keepclient/collectionreader.go              | 321 ++++++++-------------
 sdk/go/keepclient/collectionreader_test.go         |  48 ++-
 sdk/go/keepclient/keepclient.go                    |   9 +
 .../api/app/controllers/application_controller.rb  |  15 +-
 .../controllers/arvados/v1/schema_controller.rb    |  10 +-
 .../api/config/initializers/noop_deep_munge.rb     |   9 +
 .../api/test/functional/arvados/v1/query_test.rb   |  29 ++
 services/api/test/integration/noop_deep_munge.rb   |  35 +++
 services/arv-git-httpd/main.go                     |   5 +
 .../crunch-dispatch-slurm/crunch-dispatch-slurm.go |  19 +-
 services/crunch-run/crunchrun.go                   |  16 +-
 services/crunch-run/crunchrun_test.go              |  16 +-
 services/keep-balance/balance_run_test.go          |   4 +-
 services/keep-balance/collection.go                |  75 +++--
 services/keep-balance/collection_test.go           |  57 ++++
 services/keep-balance/main.go                      |  19 +-
 services/keep-web/handler.go                       |  52 +---
 services/keep-web/handler_test.go                  |  43 ---
 services/keep-web/main.go                          |   6 +
 services/keep-web/ranges_test.go                   |  90 ++++++
 services/keepproxy/keepproxy.go                    |  11 +-
 services/keepstore/keepstore.go                    |   8 +-
 55 files changed, 1320 insertions(+), 701 deletions(-)
 copy build/package-build-dockerfiles/{ubuntu1204 => ubuntu1604}/Dockerfile (72%)
 copy build/package-test-dockerfiles/{ubuntu1204 => ubuntu1604}/Dockerfile (55%)
 create mode 100644 build/package-test-dockerfiles/ubuntu1604/etc-apt-preferences.d-arvados
 rename build/package-testing/{test-packages-debian7.sh => test-packages-ubuntu1604.sh} (100%)
 copy sdk/cwl/tests/wf/{expect_packed.cwl => submit_wf_packed.cwl} (53%)
 create mode 100644 sdk/go/config/dump.go
 create mode 100644 sdk/go/keepclient/block_cache.go
 create mode 100644 services/api/config/initializers/noop_deep_munge.rb
 create mode 100644 services/api/test/integration/noop_deep_munge.rb
 create mode 100644 services/keep-balance/collection_test.go
 create mode 100644 services/keep-web/ranges_test.go

       via  da42861619eb478cd1f01d58a1ebe59f1a25002e (commit)
       via  dc7d01f4d4031962ffd5734ca0c64146a7217e4a (commit)
       via  aa05b5d5ee2e42f2e456cca0c2f3cd835ab7cef4 (commit)
       via  fdecbc701188e64dc9e539da2dddbbeded709b42 (commit)
       via  28aea3c6b1888c00a041992282c95fe595f85005 (commit)
       via  436f5c768dbc97135490b6477efd1ff0482a9dda (commit)
       via  71229e23918b698caa7c6c8b62b368d4aef2ab85 (commit)
       via  d6579d95bacbba5682d0c085f139681c93cb005a (commit)
       via  827879be023e90d58eb681b3c930154739a0b27f (commit)
       via  24b137a5b3313778e2db7f5d1e0c82daf0634a9c (commit)
       via  b001425779a0189b357c4e3a47734eda6b77ce2d (commit)
       via  2b37f7a8e843638431dc9e8031c1c2c2a81c1e7e (commit)
       via  1b8106da867aafbe6e4653485d5e644faee4c35d (commit)
       via  dcb7f718a7bf31c6e1227d28cdaac64fa769015b (commit)
       via  95d8984418a5198639000f0a622da74af259b6c5 (commit)
       via  8675eec8206c0e39753741864636c8a05ce02408 (commit)
       via  799b5333c66ccd18fe1b0ee97c2a28be6e07b731 (commit)
       via  9dd66f4d896785e0c6dce839ed1b411dd3c77bf0 (commit)
       via  7dd717ca530fccae814c8c59a8c242d376fa0cef (commit)
       via  a7222f4df954c9ac973d58bb2f27a8f049dbbbc2 (commit)
       via  a3ca6693f522b59ea50d76cadde69b91c0a62cad (commit)
       via  13c27ef91fc2516bdcdee94ba1ff2f08361dc51d (commit)
       via  5ba9471d651c9f6235988d3ec680461ba4785453 (commit)
       via  0d7ceb1e3498210d20859516de4ffdd59530b6d7 (commit)
       via  da0aee751d8cb039c9b6b85a03e7d62cb973e3b3 (commit)
       via  a20c77028c182b684a3acb791c7de4183319737e (commit)
       via  1edd524ecd7b6776dd9b20ab81030e7c69c392b4 (commit)
       via  b172bbd9e4c780d9af79774aee81a1ac48b3a857 (commit)
       via  8e569c16ba035b131c148441ca5a590fb49811ac (commit)
       via  38ac8f3e065e74f6af172cee90144701d2f73b58 (commit)
       via  1e7d93d8dcc2d896a71c38910f4fb6ef35408c0d (commit)
       via  7fd60cc870863647127a438a085685d415c37a46 (commit)
       via  aea835bc965d42e225c2641b0210c4b521f6dc4e (commit)
       via  16fe80b0e93ed8c8416b2dcbc0e2ad49bc850738 (commit)
       via  b8de9b3e62e82b806576b237be5f317bf378169f (commit)
       via  39189b90bd2f73d4f4938049ffa4441a967ca24c (commit)
       via  dc5a33cbfc156c839515acb4fa6ea2f9162a0972 (commit)
       via  cd383b7168d9412f4f097438d590e919ff7a97d6 (commit)
       via  ddcaafee5dc10ee0104c108c9648f4d5024a83d9 (commit)
       via  685af7fb2ae3a8ea162edd89eec61fdd4ca376f0 (commit)
       via  ae9f71231ed50eb35097c10c84b0070bcdcd22cf (commit)
       via  01007e0a2c7cf5461ced83339f6abcfb6f9fac72 (commit)
       via  6143bba5421756c78b282ee6c4da793d45a4523e (commit)
       via  036c59ea6b19372e74f7ccacb5dcb2f522f99629 (commit)
       via  7fa95f2db716ebfdb6312fa67b9b07bebb815b39 (commit)
       via  f40364c4d42e111b9da3873afcfaed2b49e7f182 (commit)
       via  2f953026bc4baeccb78ca82acc4d07cad37625b8 (commit)
       via  60d986b8908487c086eb4e402ac69669cb26108b (commit)
       via  c40389e0f064d4ea379d5f5471116936239a467a (commit)
       via  b6e15a581be2e5b2387fe18ecb01714fbc21d3aa (commit)
       via  80f042bec0b25966498324cb6aafb7fc24a36e12 (commit)
       via  1cdb5cbf83d34cb1a19c78478a7d5a28738e7bbf (commit)
       via  e8521124b0b8e647c58e99905100171ea5e30abd (commit)
       via  b07d8a573c74e7be56a2fd8c5064032c37d90c67 (commit)
       via  de4ecbc700759ff22e76948a58f7d70e5d3c1464 (commit)
       via  e1fd558686c78d6edfd460b7531ec9b559299889 (commit)
       via  4696633ae5e955c267d50a14e790c93d833f41b0 (commit)
       via  a9ae80084e5b8897408a47fe081481cb913fd260 (commit)
       via  609646134bcd8fc3a7fd500848220741ecc4a9d2 (commit)
       via  4032e4dc4284391f21f21630bdb7ca88f8329d8e (commit)
       via  0d0092442c009b8b3e387da18f828f3f0518b709 (commit)
       via  a4edc2a4a740d3908b907682a85e227eee80683b (commit)
       via  f65530a60d7b74da723d16eccd84576c53575401 (commit)
       via  be141b88fe871c7e72b19a5cf1e0aa56bf28283a (commit)
       via  43dcf83e8b5332004bddd01be22476e68b1cef0f (commit)
       via  bb037e3872214d2b709964405775f476c6b3b550 (commit)
      from  a5e7c0c8bf829a41c4d02b48d5a9a40d6d225c8a (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit da42861619eb478cd1f01d58a1ebe59f1a25002e
Author: radhika <radhika at curoverse.com>
Date:   Mon Jan 30 16:20:24 2017 -0500

    9397: if mount.Path ends with "/", trim it.

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 13532ae..bd4c5cc 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -763,18 +763,22 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
 			bindSubdir = "." + bindSuffix[0:bindIdx]
 			bindFileName = bindSuffix[bindIdx+1:]
 		}
-		pathIdx := strings.LastIndex(mnt.Path, "/")
+		mntPath := mnt.Path
+		if strings.HasSuffix(mntPath, "/") {
+			mntPath = mntPath[0 : len(mntPath)-1]
+		}
+		pathIdx := strings.LastIndex(mntPath, "/")
 		var pathSubdir, pathFileName string
 		if pathIdx >= 0 {
-			pathSubdir = "." + mnt.Path[0:pathIdx]
-			pathFileName = mnt.Path[pathIdx+1:]
+			pathSubdir = "." + mntPath[0:pathIdx]
+			pathFileName = mntPath[pathIdx+1:]
 		}
 		streams := strings.Split(collection.ManifestText, "\n")
 		for _, stream := range streams {
 			tokens := strings.Split(stream, " ")
-			if tokens[0] == "."+mnt.Path {
+			if tokens[0] == "."+mntPath {
 				// path refers to this complete stream
-				adjustedStream := strings.Replace(stream, "."+mnt.Path, "."+bindSuffix, -1)
+				adjustedStream := strings.Replace(stream, "."+mntPath, "."+bindSuffix, -1)
 				manifestText = adjustedStream + "\n"
 				break
 			} else {
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 76e052c..35ef013 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1219,7 +1219,7 @@ func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
         "/tmp": {"kind": "tmp"},
         "/tmp/foo": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367"},
         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt"},
-        "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1"},
+        "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/"},
         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
         "/tmp/foo/bar/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt"},
         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}

commit dc7d01f4d4031962ffd5734ca0c64146a7217e4a
Merge: a5e7c0c aa05b5d
Author: radhika <radhika at curoverse.com>
Date:   Mon Jan 30 15:59:29 2017 -0500

    Merge branch 'master' into 9397-prepopulate-output-directory


-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list