[ARVADOS] updated: c981b80d8965d62435045518f569e3f270014298
git at public.curoverse.com
git at public.curoverse.com
Fri Feb 21 13:22:57 EST 2014
Summary of changes:
.../app/views/collections/_show_files.html.erb | 2 +-
crunch_scripts/hash | 24 +-
sdk/cli/bin/arv | 4 +-
sdk/cli/bin/arv-ls | 1 +
sdk/cli/bin/arv-mount | 1 +
sdk/cli/bin/arv-normalize | 1 +
sdk/cli/bin/crunch-job | 5 +-
sdk/python/arvados/__init__.py | 13 +-
sdk/python/arvados/collection.py | 90 +++++-
sdk/python/arvados/fuse.py | 318 +++++++++++++++++++
sdk/python/arvados/keep.py | 140 +++++++--
sdk/python/arvados/stream.py | 326 +++++++++++--------
sdk/python/arvados/timer.py | 16 +
sdk/python/bin/arv-ls | 33 ++
sdk/python/bin/arv-mount | 187 +++---------
sdk/python/bin/arv-normalize | 24 ++
sdk/python/bin/arv-put | 16 +
sdk/python/readme.llfuse | 1 +
sdk/python/requirements.txt | 2 +-
sdk/python/setup.py.src | 3 +-
sdk/python/test_collections.py | 334 ++++++++++++++++++--
sdk/python/test_mount.py | 153 +++++++++
sdk/python/testdata/1000G_ref_manifest | 1 +
sdk/python/testdata/jlake_manifest | 20 ++
services/api/app/models/collection.rb | 27 ++-
25 files changed, 1365 insertions(+), 377 deletions(-)
create mode 120000 sdk/cli/bin/arv-ls
create mode 120000 sdk/cli/bin/arv-mount
create mode 120000 sdk/cli/bin/arv-normalize
create mode 100644 sdk/python/arvados/fuse.py
create mode 100644 sdk/python/arvados/timer.py
create mode 100755 sdk/python/bin/arv-ls
create mode 100755 sdk/python/bin/arv-normalize
create mode 100644 sdk/python/readme.llfuse
create mode 100644 sdk/python/test_mount.py
create mode 100644 sdk/python/testdata/1000G_ref_manifest
create mode 100644 sdk/python/testdata/jlake_manifest
via c981b80d8965d62435045518f569e3f270014298 (commit)
via 0e53f1c16b9417eecb693fe398585c81f7b4e329 (commit)
via d4346ebd1ef1158a73d9adfa757cfce0713abb1d (commit)
via 7769d5742b41712bc411d75d6cd6b536cd87a3e4 (commit)
via 06f92553e100b58d2d72a3190fb8e57abac820bb (commit)
via b6888eaa68820d4107a73fb1fea136c00866f915 (commit)
via e86a4c6a752eb72819f07cd7403b0092c00154cb (commit)
via d533fffba8615a3e4d370ed1f7cdd810cdb4eee3 (commit)
via 5f64ae1ff9649c01bc95ec86c26624730f9de724 (commit)
via 5f8c016420fa751db68409d9047d3111f7e4a12a (commit)
via bad2a6a760528915012d999466368d751c172463 (commit)
via 975c10c5b8ccc7d12ee4df993e06a359c4bbec93 (commit)
via 426b8ae13589e4ee121a094a6f18dd89a577402d (commit)
via d52c19d19d5588c664c599aaa51c0084954d8836 (commit)
via 1ce079f002566b28c450959b1d7389c8c85ab6f0 (commit)
via 2b1a709c1e3ff70d811804bfd9fa58a7231bd5cb (commit)
via 08c4e44a6a431286408ce9984bc4be84dccb49f7 (commit)
via daa5e6586560d002fd4cd9c76ea9ffc199bcb26d (commit)
via aa67d42865bb0cc31a8ee9a13a5b65b06aa3c5e4 (commit)
via fb835ba018ae9f4509063e93f7054fdd23b6669d (commit)
via d165639ac5760713b872242dd1c2eb54c34a0ffd (commit)
via 61e04bd283799308728e9c0bf085db799bd9cbc2 (commit)
via c2459c6593b1508ad01f64c4a6f91b70b3d80c89 (commit)
via f6fa860739496e94d0e08d21ba19a17c0fbea95a (commit)
via 07520e24f23c3030dc063381ed9a6163ab763a0f (commit)
via 9887cbea6421c442de1439f8c0050f81c4474dfd (commit)
via ffa11e9dc42740652da1d8382b89bc7eaaac5e2e (commit)
via e05e4cd6106b83aa21ef45a3d6537b89d7f67498 (commit)
via da939284a2a57dd046422b75a0c4700a5ff4f223 (commit)
via 444af0ae1f9b9d36af72e57ae508cd0608988236 (commit)
via 89a9429f8fe9248320fad181afe972e144d10669 (commit)
via 0b0f8c5211ee93d2fd2bceb2a4331a06cea52663 (commit)
via 600378f933f028ba497cac86a978fa71401d209b (commit)
via 43e09c03c00f14d923f7335aa3d632d7e4785fbb (commit)
via fa739e32888d42bb99cef4781dc0c3f1c6190ac7 (commit)
via a9c427bdcdb215119bb28d2b53b0f8b462048e72 (commit)
via 27d6124d152f99fc8f26cd2c9fc99d24ab46a7ea (commit)
via 519c5922c34da64a8cce9e4d0030892e9b4bdd83 (commit)
via 1b82cd274ecebba9302e8a06f6c9e99eaf8ec717 (commit)
via e0d33f390ffb609332d966fe0ce43ada6188d890 (commit)
from d9b57edd7f14f674a7fa33373471c17e3b443230 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c981b80d8965d62435045518f569e3f270014298
Merge: 0e53f1c d9b57ed
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 21 13:23:58 2014 -0500
Merge branch 'master' of git.clinicalfuture.com:arvados
# Please enter a commit message to explain why this merge is necessary,
# especially if it merges an updated upstream into a topic branch.
#
# Lines starting with '#' will be ignored, and an empty message aborts
# the commit.
diff --cc sdk/cli/bin/crunch-job
index eb6b43e,0fd8c94..e3d0c1d
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@@ -546,9 -556,9 +556,10 @@@ for (my $todo_ptr = 0; $todo_ptr <= $#j
$ENV{"TASK_SLOT_NODE"} = $slot[$childslot]->{node}->{name};
$ENV{"TASK_SLOT_NUMBER"} = $slot[$childslot]->{cpu};
$ENV{"TASK_WORK"} = $ENV{"JOB_WORK"}."/".$slot[$childslot]->{cpu};
+ $ENV{"TASK_KEEPMOUNT"} = $ENV{"TASK_WORK"}."/keep";
$ENV{"TASK_TMPDIR"} = $ENV{"TASK_WORK"}; # deprecated
$ENV{"CRUNCH_NODE_SLOTS"} = $slot[$childslot]->{node}->{ncpus};
+ $ENV{"PATH"} = $ENV{"CRUNCH_INSTALL"} . "/bin:" . $ENV{"PATH"};
$ENV{"GZIP"} = "-n";
commit 0e53f1c16b9417eecb693fe398585c81f7b4e329
Merge: bf1efb4 d4346eb
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 21 13:23:19 2014 -0500
Merge branch '1965-fuse-support-directories'
Conflicts:
sdk/python/arvados/keep.py
diff --cc sdk/cli/bin/crunch-job
index 97a4bbb,a7b88c9..eb6b43e
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@@ -571,10 -572,9 +572,10 @@@ for (my $todo_ptr = 0; $todo_ptr <= $#j
}
$ENV{"PYTHONPATH"} =~ s{^}{:} if $ENV{"PYTHONPATH"};
$ENV{"PYTHONPATH"} =~ s{^}{$ENV{CRUNCH_SRC}/sdk/python}; # xxx hack
+ $ENV{"PYTHONPATH"} =~ s{^}{$ENV{CRUNCH_SRC}/arvados/sdk/python:}; # xxx hack
$ENV{"PYTHONPATH"} =~ s{$}{:/usr/local/arvados/src/sdk/python}; # xxx hack
$command .=
- "&& exec $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
+ "&& exec arv-mount $ENV{TASK_KEEPMOUNT} --exec $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
my @execargs = ('bash', '-c', $command);
srun (\@srunargs, \@execargs, undef, $build_script_to_send);
exit (111);
diff --cc sdk/python/arvados/keep.py
index e8be158,f68b11c..88487ae
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@@ -142,15 -147,17 +147,18 @@@ class KeepClient(object)
def shuffled_service_roots(self, hash):
if self.service_roots == None:
self.lock.acquire()
- keep_disks = arvados.api().keep_disks().list().execute()['items']
- roots = (("http%s://%s:%d/" %
- ('s' if f['service_ssl_flag'] else '',
- f['service_host'],
- f['service_port']))
- for f in keep_disks)
- self.service_roots = sorted(set(roots))
- logging.debug(str(self.service_roots))
- self.lock.release()
+ try:
- keep_disks = api().keep_disks().list().execute()['items']
++ keep_disks = arvados.api().keep_disks().list().execute()['items']
+ roots = (("http%s://%s:%d/" %
+ ('s' if f['service_ssl_flag'] else '',
+ f['service_host'],
+ f['service_port']))
+ for f in keep_disks)
+ self.service_roots = sorted(set(roots))
+ logging.debug(str(self.service_roots))
+ finally:
+ self.lock.release()
++
seed = hash
pool = self.service_roots[:]
pseq = []
diff --cc services/api/app/models/collection.rb
index f6c8b26,ea18789..495b9fe
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@@ -102,22 -115,12 +119,28 @@@ class Collection < ArvadosMode
end
end
end
+
+ @files = []
+ tmp.each do |k, v|
+ re = k.match(/^(.+)\/(.+)/)
+ @files << [re[1], re[2], v]
+ end
end
end
+
+ def self.normalize_uuid uuid
+ hash_part = nil
+ size_part = nil
+ uuid.split('+').each do |token|
+ if token.match /^[0-9a-f]{32,}$/
+ raise "uuid #{uuid} has multiple hash parts" if hash_part
+ hash_part = token
+ elsif token.match /^\d+$/
+ raise "uuid #{uuid} has multiple size parts" if size_part
+ size_part = token
+ end
+ end
+ raise "uuid #{uuid} has no hash part" if !hash_part
+ [hash_part, size_part].compact.join '+'
+ end
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list