[ARVADOS] created: e289b9ad5959a76795681ca95d310bad2288656a
Git user
git at public.curoverse.com
Thu Apr 27 16:16:44 EDT 2017
at e289b9ad5959a76795681ca95d310bad2288656a (commit)
commit e289b9ad5959a76795681ca95d310bad2288656a
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Thu Apr 27 17:15:53 2017 -0300
11579: Explicitly following links on test.
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index ac8f00c..547cfc2 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -287,7 +287,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
cwriter.destroy_cache()
def test_symlinks_are_followed_only_once(self):
- cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink])
+ cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink],
+ follow_links=True)
cwriter.start(save_collection=False)
self.assertIn('linkeddir1', cwriter.manifest_text())
self.assertNotIn('linkeddir2', cwriter.manifest_text())
commit 3843210bdf340751795b8ce9903cf712661b94e7
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Thu Apr 27 17:12:56 2017 -0300
11579: Tests added.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index ff2e403..d616f30 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -433,7 +433,7 @@ class ArvPutUploadJob(object):
else:
self._traversed_links.add(real_dirpath)
return dirs
-
+
def start(self, save_collection):
"""
Start supporting thread & file uploading
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 096f86a..ac8f00c 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -270,7 +270,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
self.arvfile_write = getattr(arvados.arvfile.ArvadosFileWriter, 'write')
# Temp dir to hold a symlink to other temp dir
self.tempdir_with_symlink = tempfile.mkdtemp()
- os.symlink(self.tempdir, os.path.join(self.tempdir_with_symlink, 'linkeddir'))
+ os.symlink(self.tempdir, os.path.join(self.tempdir_with_symlink, 'linkeddir1'))
+ os.symlink(self.tempdir, os.path.join(self.tempdir_with_symlink, 'linkeddir2'))
def tearDown(self):
super(ArvPutUploadJobTest, self).tearDown()
@@ -282,7 +283,22 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
def test_symlinks_are_followed_by_default(self):
cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink])
cwriter.start(save_collection=False)
- self.assertIn('linkeddir', cwriter.manifest_text())
+ self.assertIn('linkeddir1', cwriter.manifest_text())
+ cwriter.destroy_cache()
+
+ def test_symlinks_are_followed_only_once(self):
+ cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink])
+ cwriter.start(save_collection=False)
+ self.assertIn('linkeddir1', cwriter.manifest_text())
+ self.assertNotIn('linkeddir2', cwriter.manifest_text())
+ cwriter.destroy_cache()
+
+ def test_symlinks_are_not_followed_when_requested(self):
+ cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink],
+ follow_links=False)
+ cwriter.start(save_collection=False)
+ self.assertNotIn('linkeddir1', cwriter.manifest_text())
+ self.assertNotIn('linkeddir2', cwriter.manifest_text())
cwriter.destroy_cache()
def test_writer_works_without_cache(self):
commit b92089c1674dcc99184a36b7094ac72ffc787922
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Thu Apr 27 16:51:31 2017 -0300
11579: Added symlinked dir traversal as the default behavior.
Added flag to disable this and ignore symlinked dirs: --no-follow-links.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 4251075..ff2e403 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -186,6 +186,17 @@ Do not continue interrupted uploads from cached state.
""")
_group = run_opts.add_mutually_exclusive_group()
+_group.add_argument('--follow-links', action='store_true', default=True,
+ dest='follow_links', help="""
+Traverse directory symlinks (default).
+Multiple symlinks pointing to the same directory will only be followed once.
+""")
+_group.add_argument('--no-follow-links', action='store_false', dest='follow_links',
+ help="""
+Do not traverse directory symlinks.
+""")
+
+_group = run_opts.add_mutually_exclusive_group()
_group.add_argument('--cache', action='store_true', dest='use_cache', default=True,
help="""
Save upload state in a cache file for resuming (default).
@@ -361,7 +372,8 @@ class ArvPutUploadJob(object):
ensure_unique_name=False, num_retries=None,
put_threads=None, replication_desired=None,
filename=None, update_time=60.0, update_collection=None,
- logger=logging.getLogger('arvados.arv_put'), dry_run=False):
+ logger=logging.getLogger('arvados.arv_put'), dry_run=False,
+ follow_links=True):
self.paths = paths
self.resume = resume
self.use_cache = use_cache
@@ -394,6 +406,8 @@ class ArvPutUploadJob(object):
self.logger = logger
self.dry_run = dry_run
self._checkpoint_before_quit = True
+ self.follow_links = follow_links
+ self._traversed_links = set()
if not self.use_cache and self.resume:
raise ArvPutArgumentConflict('resume cannot be True when use_cache is False')
@@ -405,6 +419,21 @@ class ArvPutUploadJob(object):
# Load cached data if any and if needed
self._setup_state(update_collection)
+ def _check_traversed_dir_links(self, root, dirs):
+ """
+ Remove from the 'dirs' list the already traversed directory symlinks,
+ register the new dir symlinks as traversed.
+ """
+ for d in [d for d in dirs if os.path.isdir(os.path.join(root, d)) and
+ os.path.islink(os.path.join(root, d))]:
+ real_dirpath = os.path.realpath(os.path.join(root, d))
+ if real_dirpath in self._traversed_links:
+ dirs.remove(d)
+ self.logger.warning("Skipping '{}' symlink to directory '{}' because it was already uploaded".format(os.path.join(root, d), real_dirpath))
+ else:
+ self._traversed_links.add(real_dirpath)
+ return dirs
+
def start(self, save_collection):
"""
Start supporting thread & file uploading
@@ -424,7 +453,12 @@ class ArvPutUploadJob(object):
prefixdir = path = os.path.abspath(path)
if prefixdir != '/':
prefixdir += '/'
- for root, dirs, files in os.walk(path):
+ # If following symlinks, avoid recursive traversals
+ if self.follow_links and os.path.islink(path):
+ self._traversed_links.add(os.path.realpath(path))
+ for root, dirs, files in os.walk(path, followlinks=self.follow_links):
+ if self.follow_links:
+ dirs = self._check_traversed_dir_links(root, dirs)
# Make os.walk()'s dir traversing order deterministic
dirs.sort()
files.sort()
@@ -798,14 +832,27 @@ class ArvPutUploadJob(object):
return datablocks
-def expected_bytes_for(pathlist):
+def expected_bytes_for(pathlist, follow_links=True):
# Walk the given directory trees and stat files, adding up file sizes,
# so we can display progress as percent
+ linked_dirs = set()
bytesum = 0
for path in pathlist:
if os.path.isdir(path):
- for filename in arvados.util.listdir_recursive(path):
- bytesum += os.path.getsize(os.path.join(path, filename))
+ for root, dirs, files in os.walk(path, followlinks=follow_links):
+ if follow_links:
+ # Skip those linked dirs that were visited more than once.
+ for d in [x for x in dirs if os.path.islink(os.path.join(root, x))]:
+ d_realpath = os.path.realpath(os.path.join(root, d))
+ if d_realpath in linked_dirs:
+ # Linked dir already visited, skip it.
+ dirs.remove(d)
+ else:
+ # Will only visit this dir once
+ linked_dirs.add(d_realpath)
+ # Sum file sizes
+ for f in files:
+ bytesum += os.path.getsize(os.path.join(root, f))
elif not os.path.isfile(path):
return None
else:
@@ -893,7 +940,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
# uploaded, the expected bytes calculation can take a moment.
if args.progress and any([os.path.isdir(f) for f in args.paths]):
logger.info("Calculating upload size, this could take some time...")
- bytes_expected = expected_bytes_for(args.paths)
+ bytes_expected = expected_bytes_for(args.paths, follow_links=args.follow_links)
try:
writer = ArvPutUploadJob(paths = args.paths,
@@ -910,7 +957,8 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
ensure_unique_name = True,
update_collection = args.update_collection,
logger=logger,
- dry_run=args.dry_run)
+ dry_run=args.dry_run,
+ follow_links=args.follow_links)
except ResumeCacheConflict:
logger.error("\n".join([
"arv-put: Another process is already uploading this data.",
commit f517c9897428fd3c50a269b494b6b912cae291df
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Thu Apr 27 13:54:32 2017 -0300
11579: Test proving the bug: arv-put does not follow symlinks.
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 286a22e..096f86a 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -268,12 +268,22 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
with open(os.path.join(self.small_files_dir, str(i)), 'w') as f:
f.write(data + str(i))
self.arvfile_write = getattr(arvados.arvfile.ArvadosFileWriter, 'write')
+ # Temp dir to hold a symlink to other temp dir
+ self.tempdir_with_symlink = tempfile.mkdtemp()
+ os.symlink(self.tempdir, os.path.join(self.tempdir_with_symlink, 'linkeddir'))
def tearDown(self):
super(ArvPutUploadJobTest, self).tearDown()
shutil.rmtree(self.tempdir)
os.unlink(self.large_file_name)
shutil.rmtree(self.small_files_dir)
+ shutil.rmtree(self.tempdir_with_symlink)
+
+ def test_symlinks_are_followed_by_default(self):
+ cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink])
+ cwriter.start(save_collection=False)
+ self.assertIn('linkeddir', cwriter.manifest_text())
+ cwriter.destroy_cache()
def test_writer_works_without_cache(self):
cwriter = arv_put.ArvPutUploadJob(['/dev/null'], resume=False)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list