[ARVADOS] updated: 1.3.0-154-g735143cbf
Git user
git at public.curoverse.com
Tue Jan 15 12:27:16 EST 2019
Summary of changes:
.licenseignore | 1 +
apps/workbench/app/helpers/application_helper.rb | 4 +
.../app/views/users/_virtual_machines.html.erb | 4 +-
apps/workbench/package-build.version | 1 +
apps/workbench/test/the.patch | 3 +
build/run-build-packages.sh | 3 +
build/run-tests.sh | 8 +
cmd/arvados-server/cmd.go | 4 +-
...oller.service => crunch-dispatch-cloud.service} | 4 +-
doc/README.textile | 2 +-
doc/Rakefile | 2 +-
doc/_config.yml | 1 +
.../ssh-access-unix.html.textile.liquid | 19 +-
lib/cloud/interfaces.go | 179 ++++++
lib/cmd/cmd.go | 5 +-
lib/dispatchcloud/cmd.go | 19 +
lib/dispatchcloud/container/queue.go | 378 ++++++++++++
lib/dispatchcloud/dispatcher.go | 197 ++++++
lib/dispatchcloud/dispatcher_test.go | 269 ++++++++
lib/dispatchcloud/driver.go | 22 +
lib/dispatchcloud/instance_set_proxy.go | 25 +
lib/dispatchcloud/logger.go | 29 +
lib/dispatchcloud/node_size.go | 71 +--
lib/dispatchcloud/readme.go | 70 +++
lib/dispatchcloud/scheduler/fix_stale_locks.go | 57 ++
lib/dispatchcloud/{ => scheduler}/gocheck_test.go | 2 +-
lib/dispatchcloud/scheduler/interfaces.go | 43 ++
lib/dispatchcloud/scheduler/run_queue.go | 165 +++++
lib/dispatchcloud/scheduler/run_queue_test.go | 318 ++++++++++
lib/dispatchcloud/scheduler/scheduler.go | 116 ++++
lib/dispatchcloud/scheduler/sync.go | 97 +++
lib/dispatchcloud/ssh_executor/executor.go | 190 ++++++
lib/dispatchcloud/ssh_executor/executor_test.go | 102 +++
lib/dispatchcloud/test/doc.go | 7 +
lib/dispatchcloud/test/fixtures.go | 28 +
lib/dispatchcloud/test/lame_instance_set.go | 118 ++++
lib/dispatchcloud/test/queue.go | 171 ++++++
lib/dispatchcloud/test/ssh_service.go | 169 +++++
lib/dispatchcloud/test/sshkey_dispatch | 27 +
lib/dispatchcloud/test/sshkey_dispatch.pub | 1 +
lib/dispatchcloud/test/sshkey_vm | 27 +
lib/dispatchcloud/test/sshkey_vm.pub | 1 +
lib/dispatchcloud/test/stub_driver.go | 318 ++++++++++
lib/dispatchcloud/{ => worker}/gocheck_test.go | 2 +-
lib/dispatchcloud/worker/pool.go | 684 +++++++++++++++++++++
lib/dispatchcloud/worker/pool_test.go | 135 ++++
lib/dispatchcloud/worker/worker.go | 320 ++++++++++
sdk/cwl/arvados_cwl/pathmapper.py | 40 ++
sdk/cwl/tests/test_pathmapper.py | 129 ++++
sdk/go/arvados/config.go | 105 +++-
sdk/go/arvados/container.go | 3 +-
sdk/go/health/aggregator_test.go | 38 +-
sdk/python/arvados/_normalize_stream.py | 10 +-
sdk/python/tests/test_collections.py | 11 +
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 2 +-
services/crunch-dispatch-slurm/node_type.go | 72 +++
services/crunch-run/background.go | 237 +++++++
services/crunch-run/crunchrun.go | 27 +
services/fuse/tests/test_mount.py | 2 +-
tools/arvbox/bin/arvbox | 18 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 4 +
tools/arvbox/lib/arvbox/docker/Dockerfile.demo | 6 +-
tools/arvbox/lib/arvbox/docker/api-setup.sh | 11 +-
tools/arvbox/lib/arvbox/docker/common.sh | 10 +-
.../service/{api => certificate}/log/main/.gitstub | 0
.../docker/service/{api => certificate}/log/run | 0
.../lib/arvbox/docker/service/certificate/run | 81 +++
.../arvbox/lib/arvbox/docker/service/composer/run | 2 +-
.../lib/arvbox/docker/service/gitolite/run-service | 2 +-
.../lib/arvbox/docker/service/nginx/run-service | 69 ++-
.../lib/arvbox/docker/service/ready/run-service | 3 +-
tools/arvbox/lib/arvbox/docker/service/ssh/run | 2 +-
.../lib/arvbox/docker/service/sso/run-service | 8 +-
.../arvbox/docker/service/websockets/run-service | 4 +-
.../arvbox/lib/arvbox/docker/service/workbench/run | 2 +
.../arvbox/docker/service/workbench/run-service | 10 +-
.../service/{api => workbench2}/log/main/.gitstub | 0
.../docker/service/{api => workbench2}/log/run | 0
.../docker/service/{composer => workbench2}/run | 2 +-
.../arvbox/docker/service/workbench2/run-service | 33 +
vendor/vendor.json | 6 +
81 files changed, 5193 insertions(+), 174 deletions(-)
create mode 100644 apps/workbench/package-build.version
create mode 100644 apps/workbench/test/the.patch
copy cmd/arvados-server/{arvados-controller.service => crunch-dispatch-cloud.service} (89%)
create mode 100644 lib/cloud/interfaces.go
create mode 100644 lib/dispatchcloud/cmd.go
create mode 100644 lib/dispatchcloud/container/queue.go
create mode 100644 lib/dispatchcloud/dispatcher.go
create mode 100644 lib/dispatchcloud/dispatcher_test.go
create mode 100644 lib/dispatchcloud/driver.go
create mode 100644 lib/dispatchcloud/instance_set_proxy.go
create mode 100644 lib/dispatchcloud/logger.go
create mode 100644 lib/dispatchcloud/readme.go
create mode 100644 lib/dispatchcloud/scheduler/fix_stale_locks.go
copy lib/dispatchcloud/{ => scheduler}/gocheck_test.go (90%)
create mode 100644 lib/dispatchcloud/scheduler/interfaces.go
create mode 100644 lib/dispatchcloud/scheduler/run_queue.go
create mode 100644 lib/dispatchcloud/scheduler/run_queue_test.go
create mode 100644 lib/dispatchcloud/scheduler/scheduler.go
create mode 100644 lib/dispatchcloud/scheduler/sync.go
create mode 100644 lib/dispatchcloud/ssh_executor/executor.go
create mode 100644 lib/dispatchcloud/ssh_executor/executor_test.go
create mode 100644 lib/dispatchcloud/test/doc.go
create mode 100644 lib/dispatchcloud/test/fixtures.go
create mode 100644 lib/dispatchcloud/test/lame_instance_set.go
create mode 100644 lib/dispatchcloud/test/queue.go
create mode 100644 lib/dispatchcloud/test/ssh_service.go
create mode 100644 lib/dispatchcloud/test/sshkey_dispatch
create mode 100644 lib/dispatchcloud/test/sshkey_dispatch.pub
create mode 100644 lib/dispatchcloud/test/sshkey_vm
create mode 100644 lib/dispatchcloud/test/sshkey_vm.pub
create mode 100644 lib/dispatchcloud/test/stub_driver.go
copy lib/dispatchcloud/{ => worker}/gocheck_test.go (90%)
create mode 100644 lib/dispatchcloud/worker/pool.go
create mode 100644 lib/dispatchcloud/worker/pool_test.go
create mode 100644 lib/dispatchcloud/worker/worker.go
create mode 100644 services/crunch-dispatch-slurm/node_type.go
create mode 100644 services/crunch-run/background.go
copy tools/arvbox/lib/arvbox/docker/service/{api => certificate}/log/main/.gitstub (100%)
copy tools/arvbox/lib/arvbox/docker/service/{api => certificate}/log/run (100%)
create mode 100755 tools/arvbox/lib/arvbox/docker/service/certificate/run
copy tools/arvbox/lib/arvbox/docker/service/{api => workbench2}/log/main/.gitstub (100%)
copy tools/arvbox/lib/arvbox/docker/service/{api => workbench2}/log/run (100%)
copy tools/arvbox/lib/arvbox/docker/service/{composer => workbench2}/run (69%)
create mode 100755 tools/arvbox/lib/arvbox/docker/service/workbench2/run-service
via 735143cbf278a87d975f71152471d59795836460 (commit)
via 29b8fe537aeb93c7974c8d8ec5f8f7cd0e67bdb3 (commit)
via 949d940b5f75114f86d381c347f0ba8f26b3e9b9 (commit)
via 3dca6408c9886ae3591486ed43410096b0a28488 (commit)
via b2b98cbc9c213d2831405b6eb37e0f52fa2e9e17 (commit)
via 36c705beb497a5ef8348e5f977ee6ad6674c7d11 (commit)
via 51c1daf863f3e1920f758f73b4e5d70ff2c706d6 (commit)
via e6bc5eeacc812ac41aadeec6fd7a907b5aab0237 (commit)
via 1b2d73e101e1fb27de201b33142c081e80bb7fc6 (commit)
via 76e3ea1d0d2f865272000a5e12f64266a90b3794 (commit)
via 9c013e7742d72254ae747948f96237698198aa85 (commit)
via 28aa8adcc90108922b1274f5568ce3c5745cca78 (commit)
via 3259e9de3d048f7a0b27e67098811640e9da8230 (commit)
via 2a6cb99cf7a21a273efe8dc793929b74149871f6 (commit)
via 55da806a494b13e88ee1b0c1e6cad211786628f3 (commit)
via fe33cfe407912df46ae34b5419ebdc84650829f6 (commit)
via 7a047d8b6fb8f5a1e0a0749d45a9e99b34a5779c (commit)
via 7252f5a926ebcd8fdf0048a4c622d7ee95cbd3b7 (commit)
via c6b8556dfb4f71c5ccc6041b48061fdb1c00c3b2 (commit)
via ee1161113168d709f0cb0513f192631c1f35b3ab (commit)
via 7b45cf3217fbee46cdd706893ec948bfedaae24c (commit)
via e787539455a4a0bf1e4f299460258905a56352d4 (commit)
via f8da36c66e4109730f7ad6f1a29f4e8cb921198c (commit)
via b3dbac2705f2bcffbf346d0b7efb85e2566b11f8 (commit)
via eabe449b9a17e14eb2ba67f6e5cb8ce1dafc8005 (commit)
via 61379d58674c14d31beab12113cd731a2dc24fd8 (commit)
via d353ff6f67923d472c1623493c7d11b35aebea90 (commit)
via cecff7f38bb5894efd1d8e449310b8654938bc15 (commit)
via e8f99cfef7cfbfcf1a1485d69250f24ced3fd609 (commit)
via a758d03b74f1c62f7ed442f4a4e571e5bba29ddd (commit)
via 772fc9527716fea221ae88ea3200db94f754afde (commit)
via cd68182c48ec70804a2acbf95331ec25629a9e28 (commit)
via 06bcc58df3e531f4af2bbc3ccf13aacdac068623 (commit)
via aab8398af23a6a01713be7c219d78bec36ffe5ad (commit)
via 32fc16adf894bb6299f94054b69ff66f133a66e7 (commit)
via 0bebb7ca32bb96b75d5996137b8e385b60f6476a (commit)
via 69e1bb1544b601e103b5fce06f3ec4cbed03d272 (commit)
via 076d8a0d4885f7bfa22962a745a5aacb48fa9be1 (commit)
via b205525d0b7c7b9042513fe77d2e8061534208ae (commit)
via e6d59d90cfbac0da3a61bb48bbca8ad85ebef129 (commit)
via 5d05b79a21ee85fa6ec53f9bc88e6ddb54029f04 (commit)
via 262871175bbe08f6ab1b7051bf9c5dfdd74283ac (commit)
via a3dd7b47276e3d509eecfb930b06309d0315ac0d (commit)
via c746a03b6b08865796b5ef722f33d4b744a9c694 (commit)
via cc952178056bf6d29471f6986306fb673dcf394a (commit)
via 10bf8762c5d882ab5e9ff4c6e316878caebc6bf8 (commit)
via 0b2772025dd5132beaebeac33d3be2e56890766e (commit)
via f6e063ed8afcdd868fe1eec2fa86ac8f0f61e18b (commit)
via df5c912a9eb5af7222e5446bc437ee97262542c8 (commit)
via 0393a615c6c795e66e021188a1a3c00c04ca68e8 (commit)
via 3877bdae211807bd4c8cc5d824ffbede288dd708 (commit)
via 0f65ce49f183f7936d694961ce0f5980a8abffcc (commit)
via ff8c3cfde1e73c1c0adccb670040e98d82120e99 (commit)
via 15b7d1d4fa484641d846c1750487e640bfe4be09 (commit)
via 93671366e7633cbf0ca3cab68395e211e3afc31c (commit)
via d6bf3b66c2c938ca2ce2bda3276c0c0f4e54f1b6 (commit)
via 49f46db107c8226d70ed6d489cd000bfba82ca2f (commit)
via 3cb857be0aad14d0ebd631305a7a5c0f847811d4 (commit)
via 3654f222637b00728158300e700d615c4c913b29 (commit)
via 1c1e245bc3163a7841859d5ff75d0cff59dfc90c (commit)
via f33f8084159f701b8a1435e8cf77b7ab7f3417ae (commit)
via e5e2dc5e6bf1bce9893517470632295d42bb5f1c (commit)
via 359dc0dba7e93744c583b431d428de62fb8e0c90 (commit)
via 17479bd75a29c52470abe0049cb447e114eb39e9 (commit)
via 67464732a18e0aac11d89a75987151543b704cf6 (commit)
via a5ef124795af5b28a97c98e69a89c4c6e99cf0b7 (commit)
via a5fe34a8cd05c4e55deaec599347f65e6d662d22 (commit)
via 8ef9e211fa1eb91deb1b6aa7245cd7e5b8dd4c33 (commit)
via 468a33e9931c3dbadc83332a3b85d7677c9e4dc3 (commit)
via 0f159e5eb304e08a0704156046762709bd6e1eba (commit)
via 3ebb991101ef62387268d594e3e4307a0f5c4e9e (commit)
via 968519edf4c9150ac65991aea5f977356551b4e4 (commit)
via 71460cf96c9b43c8ab0a38118c3745a4c0e6d7e9 (commit)
via 438ce6f86261376880b186775d7ba5fbade560a6 (commit)
via c11e1e9b7e04b819a8a1794acc3ffb0a816715b7 (commit)
via b10284a496cd2d953b865144a5f28265fc277938 (commit)
via 8dff5fe14f23f3db9efaf5d71a21668738a9f164 (commit)
via c81a653c1e800c40a0c6e1a5d94cddd6620b5e52 (commit)
via 8162f74b2611b686cc8bcec225c6724c71b8926d (commit)
via 1b47c040b802c09d8652b952704e2ad09f31aa27 (commit)
via 8c44ed01463d5e0f0d167b37474b0e78fe11bd8d (commit)
via c1b959750a78509a2f5182dbd17685d010826b64 (commit)
via c3c538444c15e68e96780f157935f2baa4ba0bc5 (commit)
from 13e7ad8135a0bafc3d1d225ff7e4c62de4f3b43f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 735143cbf278a87d975f71152471d59795836460
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Tue Jan 15 14:25:39 2019 -0300
14539: Encode \t & \n chars on file/dir names on collection's manifest.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/sdk/python/arvados/_normalize_stream.py b/sdk/python/arvados/_normalize_stream.py
index 9caef764e..f1f6052cd 100644
--- a/sdk/python/arvados/_normalize_stream.py
+++ b/sdk/python/arvados/_normalize_stream.py
@@ -8,7 +8,15 @@ from . import config
import re
def escape(path):
- return re.sub('\\\\([0-3][0-7][0-7])', lambda m: '\\134'+m.group(1), path).replace(' ', '\\040')
+ replacements = [
+ ('\t', '\\011'),
+ ('\n', '\\012'),
+ (' ', '\\040'),
+ ]
+ path = re.sub('\\\\([0-3][0-7][0-7])', lambda m: '\\134'+m.group(1), path)
+ for a, b in replacements:
+ path = path.replace(a, b)
+ return path
def normalize_stream(stream_name, stream):
"""Take manifest stream and return a list of tokens in normalized format.
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 3a4dabfea..f3ad16ecf 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -958,6 +958,17 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
c = Collection(manifest)
self.assertEqual(c.portable_manifest_text(), manifest)
+ def test_other_special_chars_on_file_token(self):
+ cases = [
+ ('\\011', '\t'),
+ ('\\012', '\n'),
+ ]
+ for encoded, decoded in cases:
+ manifest = '. d41d8cd98f00b204e9800998ecf8427e+0 0:0:some%sfile.txt\n' % encoded
+ c = Collection(manifest)
+ self.assertEqual(c.portable_manifest_text(), manifest)
+ self.assertIn('some%sfile.txt' % decoded, c.keys())
+
def test_escaped_paths_do_get_unescaped_on_listing(self):
# Dir & file names are literally '\056' (escaped form: \134056)
manifest = './\\134056\\040Test d41d8cd98f00b204e9800998ecf8427e+0 0:0:\\134056\n'
commit 29b8fe537aeb93c7974c8d8ec5f8f7cd0e67bdb3
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Tue Jan 15 11:31:43 2019 -0300
14538: Update comment.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 6a00cdb6b..bed81ad72 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -615,7 +615,7 @@ class FuseRmTest(MountTestBase):
r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
self.pool.apply(fuseRmTestHelperDeleteFile, (self.mounttmp,))
- # Can't have empty directories :-( so manifest will be empty.
+ # Empty directories are represented by an empty file named "."
collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
self.assertRegexpMatches(collection2["manifest_text"],
r'./testdir d41d8cd98f00b204e9800998ecf8427e\+0\+A\S+ 0:0:\\056\n')
commit 949d940b5f75114f86d381c347f0ba8f26b3e9b9
Merge: 13e7ad813 3dca6408c
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Jan 14 14:16:32 2019 -0300
Merge branch 'master' into 14539-pysdk-empty-dir
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list