[ARVADOS] updated: b0e6c00576257cef24cfd0599073f58904a2b85c
Git user
git at public.curoverse.com
Wed Jan 18 09:22:59 EST 2017
Summary of changes:
sdk/python/arvados/arvfile.py | 9 ++++++---
sdk/python/arvados/collection.py | 11 ++++++++---
sdk/python/arvados/commands/put.py | 32 +++++++++++++++++++++++---------
3 files changed, 37 insertions(+), 15 deletions(-)
via b0e6c00576257cef24cfd0599073f58904a2b85c (commit)
from d3ae482ebaf8c3c489f4d410ff2f5b6a550515e2 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b0e6c00576257cef24cfd0599073f58904a2b85c
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Jan 18 11:14:59 2017 -0300
10813: Bug fixed on BlockManager's _get_manifest_text() when asked for only include committed blocks on collections with subcollections inside it.
Also, added --threads parameter to arv-put command so the default number of upload threads (2) can be overridden.
Changed the update thread to execute once every minute, because it blocks the local collection for a few seconds (depending on the manifest size)
Also, cleaned up some locking uses on the update thread to minimize the time with the locking in place.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index c741ea7..9a86a8d 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -401,10 +401,10 @@ class _BlockManager(object):
"""
- DEFAULT_PUT_THREADS = 6
+ DEFAULT_PUT_THREADS = 2
DEFAULT_GET_THREADS = 2
- def __init__(self, keep, copies=None):
+ def __init__(self, keep, copies=None, put_threads=None):
"""keep: KeepClient object to use"""
self._keep = keep
self._bufferblocks = collections.OrderedDict()
@@ -414,7 +414,10 @@ class _BlockManager(object):
self._prefetch_threads = None
self.lock = threading.Lock()
self.prefetch_enabled = True
- self.num_put_threads = _BlockManager.DEFAULT_PUT_THREADS
+ if put_threads:
+ self.num_put_threads = put_threads
+ else:
+ self.num_put_threads = _BlockManager.DEFAULT_PUT_THREADS
self.num_get_threads = _BlockManager.DEFAULT_GET_THREADS
self.copies = copies
self._pending_write_size = 0
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 812438e..599e91d 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -973,7 +973,10 @@ class RichCollectionBase(CollectionBase):
if stream:
buf.append(" ".join(normalize_stream(stream_name, stream)) + "\n")
for dirname in [s for s in sorted_keys if isinstance(self[s], RichCollectionBase)]:
- buf.append(self[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip, normalize=True))
+ if only_committed:
+ buf.append(self[dirname]._get_manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip, normalize=True, only_committed=only_committed))
+ else:
+ buf.append(self[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip, normalize=True))
return "".join(buf)
else:
if strip:
@@ -1149,7 +1152,8 @@ class Collection(RichCollectionBase):
parent=None,
apiconfig=None,
block_manager=None,
- replication_desired=None):
+ replication_desired=None,
+ put_threads=None):
"""Collection constructor.
:manifest_locator_or_text:
@@ -1186,6 +1190,7 @@ class Collection(RichCollectionBase):
self._keep_client = keep_client
self._block_manager = block_manager
self.replication_desired = replication_desired
+ self.put_threads = put_threads
if apiconfig:
self._config = apiconfig
@@ -1276,7 +1281,7 @@ class Collection(RichCollectionBase):
copies = (self.replication_desired or
self._my_api()._rootDesc.get('defaultCollectionReplication',
2))
- self._block_manager = _BlockManager(self._my_keep(), copies=copies)
+ self._block_manager = _BlockManager(self._my_keep(), copies=copies, put_threads=self.put_threads)
return self._block_manager
def _remember_api_response(self, response):
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 714281c..d57ef4c 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -133,6 +133,15 @@ physical storage devices (e.g., disks) should have a copy of each data
block. Default is to use the server-provided default (if any) or 2.
""")
+upload_opts.add_argument('--threads', type=int, metavar='N', default=None,
+ help="""
+Set the number of upload threads to be used. Take into account that
+using lots of threads will increase the RAM requirements. Default is
+to use 2 threads.
+On high latency installations, using a greater number will improve
+overall troughput.
+""")
+
run_opts = argparse.ArgumentParser(add_help=False)
run_opts.add_argument('--project-uuid', metavar='UUID', help="""
@@ -347,8 +356,9 @@ class ArvPutUploadJob(object):
def __init__(self, paths, resume=True, use_cache=True, reporter=None,
bytes_expected=None, name=None, owner_uuid=None,
- ensure_unique_name=False, num_retries=None, replication_desired=None,
- filename=None, update_time=20.0, update_collection=None,
+ ensure_unique_name=False, num_retries=None,
+ put_threads=None, replication_desired=None,
+ filename=None, update_time=60.0, update_collection=None,
logger=logging.getLogger('arvados.arv_put'), dry_run=False):
self.paths = paths
self.resume = resume
@@ -363,6 +373,7 @@ class ArvPutUploadJob(object):
self.ensure_unique_name = ensure_unique_name
self.num_retries = num_retries
self.replication_desired = replication_desired
+ self.put_threads = put_threads
self.filename = filename
self._state_lock = threading.Lock()
self._state = None # Previous run state (file list & manifest)
@@ -505,14 +516,16 @@ class ArvPutUploadJob(object):
with self._collection_lock:
self.bytes_written = self._collection_size(self._local_collection)
if self.use_cache:
+ if final:
+ manifest = self._local_collection.manifest_text()
+ else:
+ # Get the manifest text without comitting pending blocks
+ manifest = self._local_collection._get_manifest_text(".", strip=False, normalize=False, only_committed=True)
# Update cache
with self._state_lock:
- if final:
- self._state['manifest'] = self._local_collection.manifest_text()
- else:
- # Get the manifest text without comitting pending blocks
- self._state['manifest'] = self._local_collection._get_manifest_text(".", strip=False, normalize=False, only_committed=True)
- self._save_state()
+ self._state['manifest'] = manifest
+ if self.use_cache:
+ self._save_state()
# Call the reporter, if any
self.report_progress()
@@ -663,7 +676,7 @@ class ArvPutUploadJob(object):
# No cache file, set empty state
self._state = copy.deepcopy(self.EMPTY_STATE)
# Load the previous manifest so we can check if files were modified remotely.
- self._local_collection = arvados.collection.Collection(self._state['manifest'], replication_desired=self.replication_desired)
+ self._local_collection = arvados.collection.Collection(self._state['manifest'], replication_desired=self.replication_desired, put_threads=self.put_threads)
def collection_file_paths(self, col, path_prefix='.'):
"""Return a list of file paths by recursively go through the entire collection `col`"""
@@ -851,6 +864,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
bytes_expected = bytes_expected,
num_retries = args.retries,
replication_desired = args.replication,
+ put_threads = args.threads,
name = collection_name,
owner_uuid = project_uuid,
ensure_unique_name = True,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list