[ARVADOS] updated: 6029fb64fd6372c577f9f143e6f3dcaded127ac3
git at public.curoverse.com
git at public.curoverse.com
Mon Jun 2 10:23:07 EDT 2014
Summary of changes:
sdk/python/arvados/commands/put.py | 20 +++--
sdk/python/test_cmdline.py | 81 -------------------
.../tests/{test_arv-put.py => test_arv_put.py} | 94 +++++++++++++++++++++-
3 files changed, 106 insertions(+), 89 deletions(-)
delete mode 100644 sdk/python/test_cmdline.py
rename sdk/python/tests/{test_arv-put.py => test_arv_put.py} (80%)
via 6029fb64fd6372c577f9f143e6f3dcaded127ac3 (commit)
via c88315d6435332f5dababac62ff4d72dadde7c67 (commit)
via e1ae50d3e62499eaa0938f27cea3e4ff8ba116a3 (commit)
from cff73e6feea770c80d9a63168de2be7cb6663a4d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6029fb64fd6372c577f9f143e6f3dcaded127ac3
Author: Brett Smith <brett at curoverse.com>
Date: Mon Jun 2 10:23:42 2014 -0400
2752: arv-put works when it can't write a cache file.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 255021e..01bae2f 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -329,7 +329,6 @@ def exit_signal_handler(sigcode, frame):
sys.exit(-sigcode)
def main(arguments=None):
- ResumeCache.setup_user_cache()
args = parse_arguments(arguments)
if args.progress:
@@ -338,17 +337,23 @@ def main(arguments=None):
reporter = progress_writer(machine_progress)
else:
reporter = None
+ bytes_expected = expected_bytes_for(args.paths)
try:
+ ResumeCache.setup_user_cache()
resume_cache = ResumeCache(ResumeCache.make_path(args))
- if not args.resume:
- resume_cache.restart()
+ except (IOError, OSError):
+ # Couldn't open cache directory/file. Continue without it.
+ resume_cache = None
+ writer = ArvPutCollectionWriter(resume_cache, reporter, bytes_expected)
except ResumeCacheConflict:
print "arv-put: Another process is already uploading this data."
sys.exit(1)
-
- writer = ArvPutCollectionWriter.from_cache(
- resume_cache, reporter, expected_bytes_for(args.paths))
+ else:
+ if not args.resume:
+ resume_cache.restart()
+ writer = ArvPutCollectionWriter.from_cache(
+ resume_cache, reporter, bytes_expected)
# Install our signal handler for each code in CAUGHT_SIGNALS, and save
# the originals.
@@ -393,7 +398,8 @@ def main(arguments=None):
for sigcode, orig_handler in orig_signal_handlers.items():
signal.signal(sigcode, orig_handler)
- resume_cache.destroy()
+ if resume_cache is not None:
+ resume_cache.destroy()
if __name__ == '__main__':
main()
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index dd720dd..bea6220 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -322,7 +322,7 @@ class ArvadosPutReportTest(ArvadosBaseTestCase):
class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
- def test_simple_file_put(self):
+ def call_main_on_test_file(self):
with self.make_test_file() as testfile:
path = testfile.name
arv_put.main(['--stream', '--no-progress', path])
@@ -331,6 +331,31 @@ class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
'098f6bcd4621d373cade4e832627b4f6')),
"did not find file stream in Keep store")
+ def test_simple_file_put(self):
+ self.call_main_on_test_file()
+
+ def test_put_with_unwriteable_cache_dir(self):
+ orig_cachedir = arv_put.ResumeCache.CACHE_DIR
+ cachedir = self.make_tmpdir()
+ os.chmod(cachedir, 0o0)
+ arv_put.ResumeCache.CACHE_DIR = cachedir
+ try:
+ self.call_main_on_test_file()
+ finally:
+ arv_put.ResumeCache.CACHE_DIR = orig_cachedir
+ os.chmod(cachedir, 0o700)
+
+ def test_put_with_unwritable_cache_subdir(self):
+ orig_cachedir = arv_put.ResumeCache.CACHE_DIR
+ cachedir = self.make_tmpdir()
+ os.chmod(cachedir, 0o0)
+ arv_put.ResumeCache.CACHE_DIR = os.path.join(cachedir, 'cachedir')
+ try:
+ self.call_main_on_test_file()
+ finally:
+ arv_put.ResumeCache.CACHE_DIR = orig_cachedir
+ os.chmod(cachedir, 0o700)
+
def test_short_put_from_stdin(self):
# Have to run this separately since arv-put can't read from the
# tests' stdin.
commit c88315d6435332f5dababac62ff4d72dadde7c67
Author: Brett Smith <brett at curoverse.com>
Date: Mon Jun 2 10:03:30 2014 -0400
2752: Rename test_arv_put.py.
This allows us to run specific test cases from the command-line. The
hyphen in the filename prevented that before.
diff --git a/sdk/python/tests/test_arv-put.py b/sdk/python/tests/test_arv_put.py
similarity index 100%
rename from sdk/python/tests/test_arv-put.py
rename to sdk/python/tests/test_arv_put.py
commit e1ae50d3e62499eaa0938f27cea3e4ff8ba116a3
Author: Brett Smith <brett at curoverse.com>
Date: Mon Jun 2 09:48:58 2014 -0400
2752/2755: Move test_cmdline code to test_arv-put.
Tim and I wrote tests for arv-put simultaneously. This commit merges
them together.
diff --git a/sdk/python/test_cmdline.py b/sdk/python/test_cmdline.py
deleted file mode 100644
index 63b6767..0000000
--- a/sdk/python/test_cmdline.py
+++ /dev/null
@@ -1,81 +0,0 @@
-import os
-import subprocess
-import unittest
-import tempfile
-import yaml
-
-import apiclient
-import arvados
-import run_test_server
-
-# ArvPutTest exercises arv-put behavior on the command line.
-#
-# Existing tests:
-#
-# ArvPutSignedManifest runs "arv-put foo" and then attempts to get
-# the newly created manifest from the API server, testing to confirm
-# that the block locators in the returned manifest are signed.
-#
-# TODO(twp): decide whether this belongs better in test_collections,
-# since it chiefly exercises behavior in arvados.collection.CollectionWriter.
-# Leaving it here for the time being because we may want to add more
-# tests for arv-put command line behavior.
-
-class ArvPutTest(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- try:
- del os.environ['KEEP_LOCAL_STORE']
- except KeyError:
- pass
-
- # Use the blob_signing_key from the Rails "test" configuration
- # to provision the Keep server.
- with open(os.path.join(os.path.dirname(__file__),
- run_test_server.ARV_API_SERVER_DIR,
- "config",
- "application.yml")) as f:
- rails_config = yaml.load(f.read())
- config_blob_signing_key = rails_config["test"]["blob_signing_key"]
- run_test_server.run()
- run_test_server.run_keep(blob_signing_key=config_blob_signing_key,
- enforce_permissions=True)
-
- @classmethod
- def tearDownClass(cls):
- run_test_server.stop()
- run_test_server.stop_keep()
-
- def test_ArvPutSignedManifest(self):
- run_test_server.authorize_with('active')
- for v in ["ARVADOS_API_HOST",
- "ARVADOS_API_HOST_INSECURE",
- "ARVADOS_API_TOKEN"]:
- os.environ[v] = arvados.config.settings()[v]
-
- # Before doing anything, demonstrate that the collection
- # we're about to create is not present in our test fixture.
- api = arvados.api('v1', cache=False)
- manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
- with self.assertRaises(apiclient.errors.HttpError):
- notfound = api.collections().get(uuid=manifest_uuid).execute()
-
- datadir = tempfile.mkdtemp()
- with open(os.path.join(datadir, "foo"), "w") as f:
- f.write("The quick brown fox jumped over the lazy dog")
- p = subprocess.Popen(["./bin/arv-put", datadir],
- stdout=subprocess.PIPE)
- (arvout, arverr) = p.communicate()
- self.assertEqual(p.returncode, 0)
- self.assertEqual(arverr, None)
- self.assertEqual(arvout.strip(), manifest_uuid)
-
- # The manifest text stored in the API server under the same
- # manifest UUID must use signed locators.
- c = api.collections().get(uuid=manifest_uuid).execute()
- self.assertRegexpMatches(
- c['manifest_text'],
- r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
-
- os.remove(os.path.join(datadir, "foo"))
- os.rmdir(datadir)
diff --git a/sdk/python/tests/test_arv-put.py b/sdk/python/tests/test_arv-put.py
index b19e1c5..dd720dd 100644
--- a/sdk/python/tests/test_arv-put.py
+++ b/sdk/python/tests/test_arv-put.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
+import apiclient
import os
import re
import shutil
@@ -9,10 +10,13 @@ import sys
import tempfile
import time
import unittest
+import yaml
import arvados
import arvados.commands.put as arv_put
+
from arvados_testutil import ArvadosBaseTestCase, ArvadosKeepLocalStoreTestCase
+import run_test_server
class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
CACHE_ARGSET = [
@@ -348,5 +352,68 @@ class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11', pipe.stdout.read())
+class ArvPutIntegrationTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ try:
+ del os.environ['KEEP_LOCAL_STORE']
+ except KeyError:
+ pass
+
+ # Use the blob_signing_key from the Rails "test" configuration
+ # to provision the Keep server.
+ with open(os.path.join(os.path.dirname(__file__),
+ run_test_server.ARV_API_SERVER_DIR,
+ "config",
+ "application.yml")) as f:
+ rails_config = yaml.load(f.read())
+ config_blob_signing_key = rails_config["test"]["blob_signing_key"]
+ run_test_server.run()
+ run_test_server.run_keep(blob_signing_key=config_blob_signing_key,
+ enforce_permissions=True)
+
+ @classmethod
+ def tearDownClass(cls):
+ run_test_server.stop()
+ run_test_server.stop_keep()
+
+ def test_ArvPutSignedManifest(self):
+ # ArvPutSignedManifest runs "arv-put foo" and then attempts to get
+ # the newly created manifest from the API server, testing to confirm
+ # that the block locators in the returned manifest are signed.
+ run_test_server.authorize_with('active')
+ for v in ["ARVADOS_API_HOST",
+ "ARVADOS_API_HOST_INSECURE",
+ "ARVADOS_API_TOKEN"]:
+ os.environ[v] = arvados.config.settings()[v]
+
+ # Before doing anything, demonstrate that the collection
+ # we're about to create is not present in our test fixture.
+ api = arvados.api('v1', cache=False)
+ manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
+ with self.assertRaises(apiclient.errors.HttpError):
+ notfound = api.collections().get(uuid=manifest_uuid).execute()
+
+ datadir = tempfile.mkdtemp()
+ with open(os.path.join(datadir, "foo"), "w") as f:
+ f.write("The quick brown fox jumped over the lazy dog")
+ p = subprocess.Popen(["./bin/arv-put", datadir],
+ stdout=subprocess.PIPE)
+ (arvout, arverr) = p.communicate()
+ self.assertEqual(p.returncode, 0)
+ self.assertEqual(arverr, None)
+ self.assertEqual(arvout.strip(), manifest_uuid)
+
+ # The manifest text stored in the API server under the same
+ # manifest UUID must use signed locators.
+ c = api.collections().get(uuid=manifest_uuid).execute()
+ self.assertRegexpMatches(
+ c['manifest_text'],
+ r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')
+
+ os.remove(os.path.join(datadir, "foo"))
+ os.rmdir(datadir)
+
+
if __name__ == '__main__':
unittest.main()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list