[ARVADOS] updated: a7b16675337995c65b909a7519178efed98a3089
git at public.curoverse.com
git at public.curoverse.com
Tue Feb 3 15:57:27 EST 2015
Summary of changes:
sdk/python/arvados/api.py | 28 --------------------
sdk/python/arvados/arvfile.py | 5 ++--
sdk/python/arvados/collection.py | 19 +++++++++-----
sdk/python/arvados/safeapi.py | 32 +++++++++++++++++++++++
sdk/python/tests/test_arvfile.py | 41 ++++++-----------------------
sdk/python/tests/test_collections.py | 50 ++++++++++++++++++++++++++++++++++++
services/fuse/bin/arv-mount | 1 +
7 files changed, 105 insertions(+), 71 deletions(-)
create mode 100644 sdk/python/arvados/safeapi.py
via a7b16675337995c65b909a7519178efed98a3089 (commit)
from 8192c879caaca60902ca362b4967cf4492d08e0c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a7b16675337995c65b909a7519178efed98a3089
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Feb 3 15:45:01 2015 -0500
4823: Add Collection.copy tests
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 0803db8..2f1f740 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -147,31 +147,3 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
svc.api_token = token
kwargs['http'].cache = None
return svc
-
-class SafeApi(object):
- """Threadsafe wrapper for API object. This stores and returns a different api
- object per thread, because httplib2 which underlies apiclient is not
- threadsafe.
- """
-
- def __init__(self, config=None, keep_params={}):
- if not config:
- config = arvados.config.settings()
- self.host = config.get('ARVADOS_API_HOST')
- self.api_token = config.get('ARVADOS_API_TOKEN')
- self.insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE')
- self.local = threading.local()
- self.keep = arvados.KeepClient(api_client=self, **keep_params)
-
- def localapi(self):
- if 'api' not in self.local.__dict__:
- self.local.api = arvados.api('v1', False, self.host,
- self.api_token, self.insecure)
- return self.local.api
-
- def __getattr__(self, name):
- # Proxy nonexistent attributes to the thread-local API client.
- try:
- return getattr(self.localapi(), name)
- except AttributeError:
- return super(SafeApi, self).__getattr__(name)
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 9d0a069..fb761e2 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -581,8 +581,7 @@ class ArvadosFile(object):
@_synchronized
def clone(self, new_parent):
"""Make a copy of this file."""
- cp = ArvadosFile()
- cp.parent = new_parent
+ cp = ArvadosFile(new_parent)
cp._modified = False
map_loc = {}
@@ -593,7 +592,7 @@ class ArvadosFile(object):
map_loc[r.locator] = self.parent._my_block_manager().dup_block(r.locator, cp).blockid
new_loc = map_loc[r.locator]
- cp.segments.append(Range(new_loc, r.range_start, r.range_size, r.segment_offset))
+ cp._segments.append(Range(new_loc, r.range_start, r.range_size, r.segment_offset))
return cp
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index dd49464..ddf2eae 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -12,6 +12,7 @@ from .arvfile import ArvadosFileBase, split, ArvadosFile, ArvadosFileWriter, Arv
from keep import *
from .stream import StreamReader, normalize_stream, locator_block_size
from .ranges import Range, LocatorAndRange
+from .safeapi import SafeApi
import config
import errors
import util
@@ -695,7 +696,7 @@ class SynchronizedCollectionBase(CollectionBase):
if p[0] == '.':
del p[0]
- if len(p) > 0:
+ if p and p[0]:
item = self._items.get(p[0])
if len(p) == 1:
# item must be a file
@@ -865,7 +866,7 @@ class SynchronizedCollectionBase(CollectionBase):
@_must_be_writable
@_synchronized
- def copyto(self, target_path, source_path, source_collection=None, overwrite=False):
+ def copy(self, source_path, target_path, source_collection=None, overwrite=False):
"""
copyto('/foo', '/bar') will overwrite 'foo' if it exists.
copyto('/foo/', '/bar') will place 'bar' in subcollection 'foo'
@@ -881,13 +882,17 @@ class SynchronizedCollectionBase(CollectionBase):
# Find parent collection the target path
tp = target_path.split("/")
- target_dir = self.find(tp[0:-1].join("/"), create=True, create_collection=True)
+ target_dir = self.find("/".join(tp[0:-1]), create=True, create_collection=True)
# Determine the name to use.
target_name = tp[-1] if tp[-1] else sp[-1]
- if target_name in target_dir and not overwrite:
- raise IOError((errno.EEXIST, "File already exists"))
+ if target_name in target_dir:
+ if isinstance(target_dir[target_name], SynchronizedCollectionBase):
+ target_dir = target_dir[target_name]
+ target_name = sp[-1]
+ elif not overwrite:
+ raise IOError((errno.EEXIST, "File already exists"))
# Actually make the copy.
dup = source_obj.clone(target_dir)
@@ -929,7 +934,7 @@ class SynchronizedCollectionBase(CollectionBase):
self[k].merge(other[k])
else:
if self[k] != other[k]:
- name = "%s~conflict-%s~" % (k, time.strftime("%Y-%m-%d~%H:%M%:%S",
+ name = "%s~conflict-%s~" % (k, time.strftime("%Y-%m-%d_%H:%M%:%S",
time.gmtime()))
self[name] = other[k].clone(self)
self.notify(self, name, ADD, self[name])
@@ -1041,7 +1046,7 @@ class Collection(SynchronizedCollectionBase):
@_synchronized
def _my_api(self):
if self._api_client is None:
- self._api_client = arvados.api.SafeApi(self._config)
+ self._api_client = arvados.SafeApi(self._config)
self._keep_client = self._api_client.keep
return self._api_client
diff --git a/sdk/python/arvados/safeapi.py b/sdk/python/arvados/safeapi.py
new file mode 100644
index 0000000..d38763c
--- /dev/null
+++ b/sdk/python/arvados/safeapi.py
@@ -0,0 +1,32 @@
+import threading
+import api
+import keep
+import config
+
+class SafeApi(object):
+ """Threadsafe wrapper for API object. This stores and returns a different api
+ object per thread, because httplib2 which underlies apiclient is not
+ threadsafe.
+ """
+
+ def __init__(self, apiconfig=None, keep_params={}):
+ if not apiconfig:
+ apiconfig = config
+ self.host = apiconfig.get('ARVADOS_API_HOST')
+ self.api_token = apiconfig.get('ARVADOS_API_TOKEN')
+ self.insecure = apiconfig.flag_is_true('ARVADOS_API_HOST_INSECURE')
+ self.local = threading.local()
+ self.keep = keep.KeepClient(api_client=self, **keep_params)
+
+ def localapi(self):
+ if 'api' not in self.local.__dict__:
+ self.local.api = api.api('v1', False, self.host,
+ self.api_token, self.insecure)
+ return self.local.api
+
+ def __getattr__(self, name):
+ # Proxy nonexistent attributes to the thread-local API client.
+ try:
+ return getattr(self.localapi(), name)
+ except AttributeError:
+ return super(SafeApi, self).__getattr__(name)
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 4bc9c2e..06c70f0 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -314,39 +314,6 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
self.assertEqual(False, c.modified())
self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
- def test_remove(self):
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
- self.assertTrue("count1.txt" in c)
- c.remove("count1.txt")
- self.assertFalse("count1.txt" in c)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
-
- def test_remove_in_subdir(self):
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
- c.remove("foo/count2.txt")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
-
- def test_remove_empty_subdir(self):
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
- c.remove("foo/count2.txt")
- c.remove("foo")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
-
- def test_remove_empty_subdir(self):
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
- with self.assertRaises(IOError):
- c.remove("foo")
- c.remove("foo", rm_r=True)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
-
- def test_prefetch(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
- with import_manifest(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep=keep) as c:
- r = c.open("count.txt", "r")
- self.assertEqual("0123", r.read(4))
- self.assertTrue("2e9ec317e197819358fbc43afca7d837+8" in keep.requests)
- self.assertTrue("e8dc4081b13434b45189a720b77b6818+8" in keep.requests)
class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
class MockParent(object):
@@ -400,6 +367,14 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
sfile.read(5)
self.assertEqual(3, sfile.tell())
+ def test_prefetch(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
+ with import_manifest(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep=keep) as c:
+ r = c.open("count.txt", "r")
+ self.assertEqual("0123", r.read(4))
+ self.assertTrue("2e9ec317e197819358fbc43afca7d837+8" in keep.requests)
+ self.assertTrue("e8dc4081b13434b45189a720b77b6818+8" in keep.requests)
+
class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
def reader_for(self, coll_name, **kwargs):
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index d9850db..892f146 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -15,6 +15,8 @@ import unittest
import run_test_server
import arvados_testutil as tutil
from arvados.ranges import Range, LocatorAndRange
+from arvados import import_manifest, export_manifest
+from arvados.arvfile import SYNC_EXPLICIT
class TestResumableWriter(arvados.ResumableCollectionWriter):
KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
@@ -820,5 +822,53 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
"""
self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", arvados.export_manifest(arvados.Collection(m1)))
+
+ def test_remove(self):
+ with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
+ self.assertTrue("count1.txt" in c)
+ c.remove("count1.txt")
+ self.assertFalse("count1.txt" in c)
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
+
+ def test_remove_in_subdir(self):
+ with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
+ c.remove("foo/count2.txt")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+
+ def test_remove_empty_subdir(self):
+ with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
+ c.remove("foo/count2.txt")
+ c.remove("foo")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+
+ def test_remove_nonempty_subdir(self):
+ with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
+ with self.assertRaises(IOError):
+ c.remove("foo")
+ c.remove("foo", rm_r=True)
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+
+ def test_copy_to_dir1(self):
+ with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT) as c:
+ c.copy("count1.txt", "foo/count2.txt")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
+
+ def test_copy_to_dir2(self):
+ with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT) as c:
+ c.copy("count1.txt", "foo")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+
+ def test_copy_to_dir2(self):
+ with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT) as c:
+ c.copy("count1.txt", "foo/")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+
+ def test_copy_file(self):
+ with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT) as c:
+ c.copy("count1.txt", "count2.txt")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/services/fuse/bin/arv-mount b/services/fuse/bin/arv-mount
index 68cd09c..1e5e9f0 100755
--- a/services/fuse/bin/arv-mount
+++ b/services/fuse/bin/arv-mount
@@ -11,6 +11,7 @@ import time
import arvados.commands._util as arv_cmd
from arvados_fuse import *
+from arvados.api import SafeApi
logger = logging.getLogger('arvados.arv-mount')
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list