[ARVADOS] updated: ddeed20d869b0d4a8dcd3c763b5dee7158f912ad
git at public.curoverse.com
git at public.curoverse.com
Mon Nov 9 11:22:36 EST 2015
Summary of changes:
doc/_config.yml | 1 +
.../install-compute-node.html.textile.liquid | 4 +
.../install-crunch-dispatch.html.textile.liquid | 2 +-
doc/install/install-keep-web.html.textile.liquid | 120 +++++++
doc/install/install-keepproxy.html.textile.liquid | 12 +-
sdk/cli/bin/crunch-job | 7 +-
sdk/cwl/arvados_cwl/__init__.py | 98 ++++--
sdk/cwl/bin/{cwl-runner => arvados-cwl-runner} | 0
sdk/cwl/setup.py | 7 +-
sdk/go/arvadosclient/arvadosclient.go | 20 ++
sdk/go/arvadosclient/arvadosclient_test.go | 78 +++--
sdk/go/arvadostest/fixtures.go | 26 ++
sdk/go/auth/auth.go | 24 ++
sdk/go/keepclient/collectionreader.go | 252 +++++++++++++++
sdk/go/keepclient/collectionreader_test.go | 223 +++++++++++++
sdk/go/keepclient/keepclient.go | 2 +-
sdk/go/keepclient/keepclient_test.go | 9 +-
sdk/go/keepclient/support.go | 38 ++-
sdk/go/manifest/manifest.go | 180 ++++++++++-
sdk/go/manifest/manifest_test.go | 81 ++++-
sdk/python/arvados/commands/run.py | 4 +-
sdk/python/tests/run_test_server.py | 2 +-
services/api/lib/create_superuser_token.rb | 48 +++
services/api/script/create_superuser_token.rb | 30 +-
.../test/fixtures/api_client_authorizations.yml | 12 +
services/api/test/fixtures/collections.yml | 8 +-
.../api/test/unit/create_superuser_token_test.rb | 78 +++++
services/crunchstat/crunchstat_test.go | 2 +-
services/datamanager/datamanager_test.go | 27 +-
services/datamanager/keep/keep.go | 2 +-
services/dockercleaner/arvados_docker/cleaner.py | 40 ++-
services/dockercleaner/tests/test_cleaner.py | 74 ++++-
services/fuse/arvados_fuse/fusedir.py | 9 +-
services/fuse/bin/arv-mount | 4 +-
services/keep-web/.gitignore | 1 +
services/keep-web/anonymous.go | 35 ++
services/keep-web/doc.go | 235 ++++++++++++++
services/keep-web/handler.go | 316 ++++++++++++++++++
services/keep-web/handler_test.go | 356 +++++++++++++++++++++
services/keep-web/main.go | 33 ++
services/{arv-git-httpd => keep-web}/server.go | 12 +-
services/keep-web/server_test.go | 322 +++++++++++++++++++
services/keepproxy/keepproxy.go | 13 +-
services/keepproxy/keepproxy_test.go | 6 +-
services/keepstore/keepstore.go | 14 +-
.../arvnodeman/computenode/dispatch/__init__.py | 22 +-
.../arvnodeman/computenode/dispatch/slurm.py | 4 +-
services/nodemanager/arvnodeman/daemon.py | 14 +-
.../nodemanager/tests/test_computenode_dispatch.py | 14 +-
services/nodemanager/tests/test_daemon.py | 20 ++
tools/keep-rsync/keep-rsync_test.go | 10 +-
51 files changed, 2756 insertions(+), 195 deletions(-)
create mode 100644 doc/install/install-keep-web.html.textile.liquid
copy sdk/cwl/bin/{cwl-runner => arvados-cwl-runner} (100%)
create mode 100644 sdk/go/arvadostest/fixtures.go
create mode 100644 sdk/go/keepclient/collectionreader.go
create mode 100644 sdk/go/keepclient/collectionreader_test.go
create mode 100755 services/api/lib/create_superuser_token.rb
create mode 100644 services/api/test/unit/create_superuser_token_test.rb
create mode 100644 services/keep-web/.gitignore
create mode 100644 services/keep-web/anonymous.go
create mode 100644 services/keep-web/doc.go
create mode 100644 services/keep-web/handler.go
create mode 100644 services/keep-web/handler_test.go
create mode 100644 services/keep-web/main.go
copy services/{arv-git-httpd => keep-web}/server.go (52%)
create mode 100644 services/keep-web/server_test.go
via ddeed20d869b0d4a8dcd3c763b5dee7158f912ad (commit)
via 9bc0194676e3f22b41976fefc6146d7dd965d173 (commit)
via 4ec9919ea6cce4ada252f1e0a0dc521fe27e508c (commit)
via 49fccd5b00230418a0918a33d4fb9154af63220a (commit)
via 8e1e7e6bf355eb0f1defc7278f1434c393a75a75 (commit)
via 05a0e777cdef513f5365d5319fea7af01ae9e7f7 (commit)
via b85cb1ba40385444f9494bbb88248ab65d700c85 (commit)
via 28425a5cf55a26113bdbea53381f86b779d84bf9 (commit)
via 6a8d940762f1b9be1a3a273e13b070d1f75ef8f1 (commit)
via fa96bcea706312be52f1488da3dad34452f5ccf4 (commit)
via 6b606a655410d9ed5116c3a3b6b6971bd90f9666 (commit)
via c74d29f1a53a1020520ffe6992ce4a39a350b5ef (commit)
via 57d3837ca9a8c5e4e71f9741931cc67e908fb3dd (commit)
via 0a43beae9c546b356220b281a432e1fa1986a305 (commit)
via 21bcf992a7e4eb806ea45c851356b2a50015fa65 (commit)
via 3c0ef3fc0cdb3f56d90442cc389ce4c2c94e8831 (commit)
via 3bef97ac46da8f5a7c322ec31c6bba4affb8ad80 (commit)
via 0a365f3d1a19fc27b402c9c742debf8975424b3f (commit)
via 242d3272391baeb95eb0a5e4e51627c2d54e7bc6 (commit)
via 59b22379a693d1fc15b4b77e58e47bac89e62660 (commit)
via 1d1c6de3c842a33a57b7d469fdaaaa1b873433dc (commit)
via 21006cfaf6d4d0ac3884a72803a8723bc4bb76fb (commit)
via 78af1220d9e2ddf4d933d9a9487397414d8a3909 (commit)
via 8020be1d7230470b9052b11740bfed08e75029de (commit)
via 97128f31a4454621329ea12e031e88226a46d586 (commit)
via efb5501e40de0d49c285db0d7a22ff8a8593832c (commit)
via df25254be20988a99061cdaced053a17b0dc9bf9 (commit)
via 0dab1775b07d67ca25974c5b68189717b9f8952f (commit)
via 62817f027167fbe5fc5b66bbdc04261da16fe3ef (commit)
via 6207681aa301ad12d164aab24f52a210945af04b (commit)
via 9b48b17eddea5e366e0c59ed9f3540793550256c (commit)
via 82b7adbaf524d6e7fd2b9a6403f9e490dcb3ac85 (commit)
via bd9722cb7b6f92b752ac786797d884ea711944cb (commit)
via 7bde93de3260e08503c7bf9c06fd3448972d378a (commit)
via 4c2e89e152736ca977fc5c2fd1d4b58c4e87720f (commit)
via 1057d0912fef22aeaa707fd428521a76806a21d0 (commit)
via e10ccaba824b4f60ddc516903304351496b5fdca (commit)
via b11cc5c782e4e94a422b8133be42bbb263d48dba (commit)
via 7bf5f28cec1ff7be1e66925c3815c253989f9eb1 (commit)
via 87640a81e725d0246837994acbb3a696d14401c6 (commit)
via 4103db7eade3ce5e06889adf12264fa03980d9d7 (commit)
via 8454069c654cdf3fb26d129cd27a4fbfbe8dfa3c (commit)
via 263cd68f5ae4b114d3c1c89f84be46b0f64f9c9e (commit)
via 6e76e3322d66f609dabcd34c98cba34bd739e089 (commit)
via e3a1bf022cbc081cf7c7ebb441dc1ab1df200f98 (commit)
via f0fdf6e7bdfd252b22c6347c292ddd511bfa31eb (commit)
via a58956e93618463e01733acaa9e8d6ce0afbc5ab (commit)
via 8d3305862395e930a8a40383871426cfacebeab6 (commit)
via 2411624900867c1ca3884e84a6ad07e30b5d0802 (commit)
via 6cef86f2d748be96695293b48afac61e2ae640df (commit)
via 585bd329a7aea0cad03b99431e14fc6ca919a76f (commit)
via a5e86befc352e31600399e9c4a476479d2155413 (commit)
via 0f668000c8cacec50804cb0019dbe7d7dc1d2b36 (commit)
via a8caa6e1b359893e99226a8b21f356eee6edd529 (commit)
via c830508697da4bf896be9ea7cea9c9cbfc5709b6 (commit)
via 8f237f88d58d9634814010e00ffe55e2cdde2334 (commit)
via a62773375ef74bbdd6a77f5dfa2023e122e717db (commit)
via 405c8505ccf36f7a1cfec68f305fae2edc439fa0 (commit)
via c33fa89b10dad03c8dce3459358ff7fe4825aa99 (commit)
via a0b3480b644c79a44f3ad03030b573d40111fbaf (commit)
via 001e1ecfddb4f1d0cc26cdb415395ff47cde4914 (commit)
via c7137167b17cb07b2d2aca325fe028c54006741d (commit)
via e9153135c39388bf403ea94896f935ce80309b01 (commit)
via 07ad618c10f03f9d24970670f991791d4bd22b62 (commit)
via 5f822d9281e5afea05b1c87e59740687c0ee6692 (commit)
via 831b63c1b17650fb28112360c5e123d91c7d546b (commit)
via 3db04ce738a8e032eda2ebf8fe1acfa3ced41bbd (commit)
via ead2387d5dbbf15065d0ec07a3a4982628fae995 (commit)
via c8f07fc798c13ad8a45fb322eda4a922f664bb4c (commit)
via 6e19c8ac3af42ad249d6e2b48b93b629e43dc79b (commit)
via e333aa8b481c3347a3543eb108a622dfef5b30b6 (commit)
via d8a8e73d39fb5fe7bd4f23b2da11e5384299489d (commit)
via a3b118be7cfb7df89d4b236a6fa4d70963fc4931 (commit)
via 7f26842ce8dbc3933086a824648969d1b90555f3 (commit)
via bbc35c5bc1ae72ef2b635292f490f2b0cf3f5475 (commit)
via f9bd0dc5aabd8ec7d811688cfb06c275a296acd5 (commit)
via d4f30dc6f8ef842e716cf37b850ab9b4fa45869a (commit)
via 6535e3ebbb702bc73a031b3b4a4cc4987729dedb (commit)
via 4f245ddd7f4cb4e10dc0d844a78e809c0cd97af7 (commit)
via c47e155f5f14a6563151b3a41374a4f547d10f0b (commit)
via 4f5a6df52559b90d2c9412624f3c4c7fbe467579 (commit)
via 477ff5db1a0cae68c75b045f0597c12ee06f67c9 (commit)
via 11890d971215ec44b7abb1e24ad748aefa1061a7 (commit)
via 2d45dab88989fcaa157b44a014c889d664919672 (commit)
via b65d8b9008c4d0e6b5816d21bf6f1ae81167ee56 (commit)
via df9cb8ad02aaee8045cb31e207fd9c6a13c01684 (commit)
via 96ce48a816ce1857e1ca3d035b3ab9002b7bc4c4 (commit)
via 8841e6b3fb247d3b3342379c86c0231d246731ff (commit)
via 0e7e5ba38ac7e86e0e398df063f612d84927692a (commit)
from 64027668d7b5bd47a863c86277c83961d8a29914 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ddeed20d869b0d4a8dcd3c763b5dee7158f912ad
Author: radhika <radhika at curoverse.com>
Date: Mon Nov 9 11:21:35 2015 -0500
7661: rename MagiDirectory by_pdh as pdh_only
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 5856888..c078242 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -496,11 +496,11 @@ point the collection will actually be looked up on the server and the directory
will appear if it exists.
""".lstrip()
- def __init__(self, parent_inode, inodes, api, num_retries, by_pdh=False):
+ def __init__(self, parent_inode, inodes, api, num_retries, pdh_only=False):
super(MagicDirectory, self).__init__(parent_inode, inodes)
self.api = api
self.num_retries = num_retries
- self.by_pdh = by_pdh
+ self.pdh_only = pdh_only
def __setattr__(self, name, value):
super(MagicDirectory, self).__setattr__(name, value)
@@ -518,10 +518,7 @@ will appear if it exists.
if k in self._entries:
return True
- if self.by_pdh and uuid_pattern.match(k):
- raise llfuse.FUSEError(errno.ENOENT)
-
- if not portable_data_hash_pattern.match(k) and not uuid_pattern.match(k):
+ if not portable_data_hash_pattern.match(k) and (self.pdh_only or not uuid_pattern.match(k)):
return False
try:
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index e2df1c8..81f8ca6 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -32,7 +32,7 @@ with "--".
mount_mode = parser.add_mutually_exclusive_group()
- mount_mode.add_argument('--all', action='store_true', help="""Mount a subdirectory for each mode: home, shared, by_tag, by_pdh, by_id (default).""")
+ mount_mode.add_argument('--all', action='store_true', help="""Mount a subdirectory for each mode: home, shared, by_tag, by_id (default).""")
mount_mode.add_argument('--home', action='store_true', help="""Mount only the user's home project.""")
mount_mode.add_argument('--shared', action='store_true', help="""Mount only list of projects shared with the user.""")
mount_mode.add_argument('--by-tag', action='store_true',
@@ -132,7 +132,7 @@ with "--".
e = operations.inodes.add_entry(Directory(llfuse.ROOT_INODE, operations.inodes))
dir_args[0] = e.inode
- e._entries['by_id'] = operations.inodes.add_entry(MagicDirectory(*dir_args, by_pdh=True if args.by_pdh else False))
+ e._entries['by_id'] = operations.inodes.add_entry(MagicDirectory(*dir_args, pdh_only=True if args.by_pdh else False))
e._entries['by_tag'] = operations.inodes.add_entry(TagsDirectory(*dir_args))
commit 9bc0194676e3f22b41976fefc6146d7dd965d173
Merge: 6402766 4ec9919
Author: radhika <radhika at curoverse.com>
Date: Mon Nov 9 10:43:13 2015 -0500
Merge branch 'master' into 7661-fuse-by-pdh
diff --cc sdk/cli/bin/crunch-job
index a8de76b,c2ea186..e2a4e26
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@@ -893,12 -893,13 +893,13 @@@ for (my $todo_ptr = 0; $todo_ptr <= $#j
.q{&& SWAP=$(awk '($1 == "SwapTotal:"){print $2}' </proc/meminfo) }
."&& MEMLIMIT=\$(( (\$MEM * 95) / ($ENV{CRUNCH_NODE_SLOTS} * 100) )) "
."&& let SWAPLIMIT=\$MEMLIMIT+\$SWAP ";
- $command .= "&& exec arv-mount --by-id --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
+ $command .= "&& exec arv-mount --by-pdh --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
if ($docker_hash)
{
- my $cidfile = "$ENV{CRUNCH_TMP}/$Jobstep->{arvados_task}->{uuid}-$Jobstep->{failures}.cid";
+ my $containername = "$Jobstep->{arvados_task}->{uuid}-$Jobstep->{failures}";
+ my $cidfile = "$ENV{CRUNCH_TMP}/$containername.cid";
$command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$cidfile -poll=10000 ";
- $command .= "$docker_bin run --rm=true --attach=stdout --attach=stderr --attach=stdin -i \Q$dockeruserarg\E --cidfile=$cidfile --sig-proxy ";
+ $command .= "$docker_bin run --name=$containername --attach=stdout --attach=stderr --attach=stdin -i \Q$dockeruserarg\E --cidfile=$cidfile --sig-proxy ";
# We only set memory limits if Docker lets us limit both memory and swap.
# Memory limits alone have been supported longer, but subprocesses tend
# to get SIGKILL if they exceed that without any swap limit set.
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list