[ARVADOS] updated: 159f578decf491f23961440d9b4e60f32f2fc231
git at public.curoverse.com
git at public.curoverse.com
Thu May 29 13:11:23 EDT 2014
Summary of changes:
.../app/assets/stylesheets/collections.css.scss | 8 ++--
.../app/controllers/collections_controller.rb | 42 +++++++++++++++++
apps/workbench/app/models/arvados_api_client.rb | 8 +++-
.../app/views/collections/_sharing_button.html.erb | 21 +++++++++
.../app/views/collections/_show_files.html.erb | 7 ++-
.../app/views/collections/sharing_popup.js.erb | 1 +
apps/workbench/app/views/collections/show.html.erb | 8 +++-
apps/workbench/config/routes.rb | 3 ++
sdk/go/src/arvados.org/keepclient/support.go | 7 +++
sdk/python/arvados/commands/put.py | 52 +++++++++++++++-------
sdk/python/tests/test_arv-put.py | 5 ++-
.../v1/api_client_authorizations_controller.rb | 2 +
services/api/app/models/arvados_model.rb | 4 +-
.../keep/src/arvados.org/keepproxy/keepproxy.go | 1 +
14 files changed, 146 insertions(+), 23 deletions(-)
create mode 100644 apps/workbench/app/views/collections/_sharing_button.html.erb
create mode 100644 apps/workbench/app/views/collections/sharing_popup.js.erb
via 159f578decf491f23961440d9b4e60f32f2fc231 (commit)
via a3acec2c35d4cf6f16e39375bc668ffe29e8d875 (commit)
via 3e555d75efb63c66ec6e85d376dc85e7447c64cc (commit)
via ba1b5732843a1b78c11fc311a52128acc1fb9f6d (commit)
via e947468785cfcd8ebd1324ca1b477351a55b10fd (commit)
via 317064a4ddead0d64d6e312a21d2bb34504aa104 (commit)
via 82b46502f25b6992c93bfe7689acc095aa447e5b (commit)
via 19f4e54627a88c115e299fa328acf22504d1ce66 (commit)
via ec07cd1ad893c15d94844b3cf2d8d95ae5cfd611 (commit)
via 747aa7b4d080b4ea95bf7d6d8643c43e70966f33 (commit)
via 651638a28db20a2016dff02e3baa106ab27ff945 (commit)
via 1db007eb53d0401a7a0ba168add7c4a094790fa5 (commit)
via 675794872a5d064cf0a8177d662555c04b0dae51 (commit)
via 06a0c1d9f2e6cf1d5a9fd00b53071d857252f9fa (commit)
via 8b030cb82d414bfa0559a205c150f4bfe792caba (commit)
via 871a7250874ec52543bed51c8b6d14a3ab860eb8 (commit)
via c3fd48d0728c140fbe0ab038ad148cfae8104c97 (commit)
via 35bc4e20adcc706ffdda3b1c9aeed1b34a20c51b (commit)
from 891d6b276f1e37d85d1a66496e045cd7ab6807e1 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 159f578decf491f23961440d9b4e60f32f2fc231
Merge: a3acec2 ba1b573
Author: Brett Smith <brett at curoverse.com>
Date: Thu May 29 13:11:36 2014 -0400
Merge branch 'master' into 2752-arv-put-resume-wip
commit a3acec2c35d4cf6f16e39375bc668ffe29e8d875
Author: Brett Smith <brett at curoverse.com>
Date: Thu May 29 13:07:23 2014 -0400
2752: arv-put only caches state on failure.
This is slightly less robust, but writing the cache after every data
flush is too expensive.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 44f911e..8667026 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -250,9 +250,7 @@ class ArvPutCollectionWriter(arvados.ResumableCollectionWriter):
print >>sys.stderr, "arv-put: Resuming previous upload. Bypass with the --no-resume option."
self.report_progress(self.bytes_written, self.bytes_expected)
- def checkpoint_state(self):
- if self.cache is None:
- return
+ def cache_state(self):
state = self.dump_state()
# Transform attributes for serialization.
for attr, value in state.items():
@@ -340,17 +338,21 @@ def main(arguments=None):
print "arv-put: Another process is already uploading this data."
sys.exit(1)
- writer = ArvPutCollectionWriter.from_cache(
- resume_cache, reporter, expected_bytes_for(args.paths))
-
- # Copy file data to Keep.
- for path in args.paths:
- if os.path.isdir(path):
- writer.write_directory_tree(
- path, max_manifest_depth=args.max_manifest_depth)
- else:
- writer.start_new_stream()
- writer.write_file(path, args.filename or os.path.basename(path))
+ try:
+ writer = ArvPutCollectionWriter.from_cache(
+ resume_cache, reporter, expected_bytes_for(args.paths))
+
+ # Copy file data to Keep.
+ for path in args.paths:
+ if os.path.isdir(path):
+ writer.write_directory_tree(
+ path, max_manifest_depth=args.max_manifest_depth)
+ else:
+ writer.start_new_stream()
+ writer.write_file(path, args.filename or os.path.basename(path))
+ except (Exception, KeyboardInterrupt):
+ writer.cache_state()
+ raise
if args.stream:
print writer.manifest_text(),
diff --git a/sdk/python/tests/test_arv-put.py b/sdk/python/tests/test_arv-put.py
index 9623923..dde42e6 100644
--- a/sdk/python/tests/test_arv-put.py
+++ b/sdk/python/tests/test_arv-put.py
@@ -199,6 +199,7 @@ class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
def test_writer_caches(self):
cwriter = arv_put.ArvPutCollectionWriter(self.cache)
cwriter.write_file('/dev/null')
+ cwriter.cache_state()
self.assertTrue(self.cache.load())
self.assertEquals(". 0:0:null\n", cwriter.manifest_text())
@@ -211,6 +212,7 @@ class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
cwriter = arv_put.ArvPutCollectionWriter(self.cache)
with self.make_test_file() as testfile:
cwriter.write_file(testfile.name, 'test')
+ cwriter.cache_state()
new_writer = arv_put.ArvPutCollectionWriter.from_cache(
self.cache)
self.assertEquals(
@@ -235,6 +237,7 @@ class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
# These bytes are intentionally not valid UTF-8.
with self.make_test_file('\x00\x07\xe2') as testfile:
cwriter.write_file(testfile.name, 'test')
+ cwriter.cache_state()
new_writer = arv_put.ArvPutCollectionWriter.from_cache(
self.cache)
self.assertEquals(cwriter.manifest_text(), new_writer.manifest_text())
@@ -261,7 +264,7 @@ class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
# Set up a writer with some flushed bytes.
cwriter.write_file(testfile.name, 'test')
cwriter.finish_current_stream()
- cwriter.checkpoint_state()
+ cwriter.cache_state()
# Restore a writer from that state and check its progress report.
# We're also checking that progress is reported immediately after
# resuming.
commit 3e555d75efb63c66ec6e85d376dc85e7447c64cc
Author: Brett Smith <brett at curoverse.com>
Date: Thu May 29 13:02:14 2014 -0400
2752: Don't duplicate arv-put work after resume.
This change serializes the command-line arguments that we've actually
processed, vs. not. That allows us to safely iterate over them again
to upload any files that we hadn't started, while skipping the ones
we've already done.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 1ccf786..44f911e 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -221,10 +221,11 @@ class ResumeCache(object):
class ArvPutCollectionWriter(arvados.ResumableCollectionWriter):
STATE_PROPS = (arvados.ResumableCollectionWriter.STATE_PROPS +
- ['bytes_written'])
+ ['bytes_written', '_seen_inputs'])
def __init__(self, cache=None, reporter=None, bytes_expected=None):
self.bytes_written = 0
+ self._seen_inputs = []
self.cache = cache
if reporter is None:
self.report_progress = lambda bytes_w, bytes_e: None
@@ -267,6 +268,25 @@ class ArvPutCollectionWriter(arvados.ResumableCollectionWriter):
self.bytes_written += (bytes_buffered - self._data_buffer_len)
self.report_progress(self.bytes_written, self.bytes_expected)
+ def _record_new_input(self, input_type, source_name, dest_name):
+ # The key needs to be a list because that's what we'll get back
+ # from JSON deserialization.
+ key = [input_type, source_name, dest_name]
+ if key in self._seen_inputs:
+ return False
+ self._seen_inputs.append(key)
+ return True
+
+ def write_file(self, source, filename=None):
+ if self._record_new_input('file', source, filename):
+ super(ArvPutCollectionWriter, self).write_file(source, filename)
+
+ def write_directory_tree(self,
+ path, stream_name='.', max_manifest_depth=-1):
+ if self._record_new_input('directory', path, stream_name):
+ super(ArvPutCollectionWriter, self).write_directory_tree(
+ path, stream_name, max_manifest_depth)
+
def expected_bytes_for(pathlist):
# Walk the given directory trees and stat files, adding up file sizes,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list