[ARVADOS] updated: 891d6b276f1e37d85d1a66496e045cd7ab6807e1
git at public.curoverse.com
git at public.curoverse.com
Thu May 29 09:39:16 EDT 2014
Summary of changes:
.../src/arvados.org/keepclient/keepclient_test.go | 2 +-
sdk/python/arvados/collection.py | 4 +++
sdk/python/arvados/commands/put.py | 20 +++++++-----
sdk/python/tests/run_test_server.py | 38 ++++++++++++++--------
sdk/python/tests/test_arv-put.py | 15 +++++----
.../src/arvados.org/keepproxy/keepproxy_test.go | 2 +-
6 files changed, 49 insertions(+), 32 deletions(-)
via 891d6b276f1e37d85d1a66496e045cd7ab6807e1 (commit)
via ce5813a53eda0f1d1512f6c080e6c4ed95179089 (commit)
via 13345f2805097df138faf7fcd1f0ed31a0d948fb (commit)
via 1a37f499f5b19fbbf508f5f9a0357a20d03d0f7e (commit)
from 2ddb3dfd39f71e606d1cb51f3924d62db13a4793 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 891d6b276f1e37d85d1a66496e045cd7ab6807e1
Author: Brett Smith <brett at curoverse.com>
Date: Thu May 29 09:28:30 2014 -0400
2752: Adapt Go tests to the new Python test infrastructure.
Refs #2752. There's two pieces to this: pointing the Go tests at the
new home of run_test_server.py, and having run_test_server.py add the
arvados module source to sys.path as needed.
diff --git a/sdk/go/src/arvados.org/keepclient/keepclient_test.go b/sdk/go/src/arvados.org/keepclient/keepclient_test.go
index 8eedadd..753a0ac 100644
--- a/sdk/go/src/arvados.org/keepclient/keepclient_test.go
+++ b/sdk/go/src/arvados.org/keepclient/keepclient_test.go
@@ -36,7 +36,7 @@ type StandaloneSuite struct{}
func pythonDir() string {
gopath := os.Getenv("GOPATH")
- return fmt.Sprintf("%s/../python", strings.Split(gopath, ":")[0])
+ return fmt.Sprintf("%s/../python/tests", strings.Split(gopath, ":")[0])
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 5b4097d..dc95d8a 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -1,14 +1,24 @@
-import subprocess
-import time
+#!/usr/bin/env python
+
+import argparse
import os
+import shutil
import signal
-import yaml
+import subprocess
import sys
-import argparse
-import arvados.config
-import arvados.api
-import shutil
import tempfile
+import time
+import yaml
+
+MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
+if __name__ == '__main__' and os.path.exists(
+ os.path.join(MY_DIRNAME, '..', 'arvados', '__init__.py')):
+ # We're being launched to support another test suite.
+ # Add the Python SDK source to the library path.
+ sys.path.insert(1, os.path.dirname(MY_DIRNAME))
+
+import arvados.api
+import arvados.config
ARV_API_SERVER_DIR = '../../../services/api'
KEEP_SERVER_DIR = '../../../services/keep'
@@ -54,7 +64,7 @@ def kill_server_pid(PID_PATH, wait=10):
def run(websockets=False, reuse_server=False):
cwd = os.getcwd()
- os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
+ os.chdir(os.path.join(MY_DIRNAME, ARV_API_SERVER_DIR))
if websockets:
pid_file = WEBSOCKETS_SERVER_PID_PATH
@@ -106,7 +116,7 @@ def run(websockets=False, reuse_server=False):
def stop():
cwd = os.getcwd()
- os.chdir(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR))
+ os.chdir(os.path.join(MY_DIRNAME, ARV_API_SERVER_DIR))
kill_server_pid(WEBSOCKETS_SERVER_PID_PATH, 0)
kill_server_pid(SERVER_PID_PATH, 0)
@@ -144,7 +154,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
stop_keep()
cwd = os.getcwd()
- os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+ os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR))
if os.environ.get('GOPATH') == None:
os.environ["GOPATH"] = os.getcwd()
else:
@@ -194,7 +204,7 @@ def _stop_keep(n):
def stop_keep():
cwd = os.getcwd()
- os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+ os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR))
_stop_keep(0)
_stop_keep(1)
@@ -205,7 +215,7 @@ def run_keep_proxy(auth):
stop_keep_proxy()
cwd = os.getcwd()
- os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+ os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR))
if os.environ.get('GOPATH') == None:
os.environ["GOPATH"] = os.getcwd()
else:
@@ -232,13 +242,13 @@ def run_keep_proxy(auth):
def stop_keep_proxy():
cwd = os.getcwd()
- os.chdir(os.path.join(os.path.dirname(__file__), KEEP_SERVER_DIR))
+ os.chdir(os.path.join(MY_DIRNAME, KEEP_SERVER_DIR))
kill_server_pid("tmp/keepproxy.pid", 0)
os.chdir(cwd)
def fixture(fix):
'''load a fixture yaml file'''
- with open(os.path.join(os.path.dirname(__file__), ARV_API_SERVER_DIR, "test", "fixtures",
+ with open(os.path.join(MY_DIRNAME, ARV_API_SERVER_DIR, "test", "fixtures",
fix + ".yml")) as f:
return yaml.load(f.read())
diff --git a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
index 9e78223..47c33b4 100644
--- a/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
+++ b/services/keep/src/arvados.org/keepproxy/keepproxy_test.go
@@ -31,7 +31,7 @@ type ServerRequiredSuite struct{}
func pythonDir() string {
gopath := os.Getenv("GOPATH")
- return fmt.Sprintf("%s/../../sdk/python", strings.Split(gopath, ":")[0])
+ return fmt.Sprintf("%s/../../sdk/python/tests", strings.Split(gopath, ":")[0])
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
commit ce5813a53eda0f1d1512f6c080e6c4ed95179089
Author: Brett Smith <brett at curoverse.com>
Date: Thu May 29 08:55:25 2014 -0400
2752: arv-put displays progress quickly after resuming.
Refs #2752.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index d67cf84..1ccf786 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -226,7 +226,10 @@ class ArvPutCollectionWriter(arvados.ResumableCollectionWriter):
def __init__(self, cache=None, reporter=None, bytes_expected=None):
self.bytes_written = 0
self.cache = cache
- self.report_func = reporter
+ if reporter is None:
+ self.report_progress = lambda bytes_w, bytes_e: None
+ else:
+ self.report_progress = reporter
self.bytes_expected = bytes_expected
super(ArvPutCollectionWriter, self).__init__()
@@ -244,6 +247,7 @@ class ArvPutCollectionWriter(arvados.ResumableCollectionWriter):
def preresume_hook(self):
print >>sys.stderr, "arv-put: Resuming previous upload. Bypass with the --no-resume option."
+ self.report_progress(self.bytes_written, self.bytes_expected)
def checkpoint_state(self):
if self.cache is None:
@@ -261,8 +265,7 @@ class ArvPutCollectionWriter(arvados.ResumableCollectionWriter):
bytes_buffered = self._data_buffer_len
super(ArvPutCollectionWriter, self).flush_data()
self.bytes_written += (bytes_buffered - self._data_buffer_len)
- if self.report_func is not None:
- self.report_func(self.bytes_written, self.bytes_expected)
+ self.report_progress(self.bytes_written, self.bytes_expected)
def expected_bytes_for(pathlist):
diff --git a/sdk/python/tests/test_arv-put.py b/sdk/python/tests/test_arv-put.py
index 4b7dc89..9623923 100644
--- a/sdk/python/tests/test_arv-put.py
+++ b/sdk/python/tests/test_arv-put.py
@@ -263,10 +263,11 @@ class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
cwriter.finish_current_stream()
cwriter.checkpoint_state()
# Restore a writer from that state and check its progress report.
+ # We're also checking that progress is reported immediately after
+ # resuming.
progression, reporter = self.make_progress_tester()
new_writer = arv_put.ArvPutCollectionWriter.from_cache(
self.cache, reporter, bytes_expected=4)
- new_writer.flush_data()
self.assertIn((4, 4), progression)
commit 13345f2805097df138faf7fcd1f0ed31a0d948fb
Author: Brett Smith <brett at curoverse.com>
Date: Thu May 29 08:54:33 2014 -0400
2752: arv-put displays a notice when it resumes.
Based on a suggestion from Peter in refs #2752.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 41adc88..f339381 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -409,9 +409,13 @@ class ResumableCollectionWriter(CollectionWriter):
except IOError as error:
raise errors.StaleWriterStateError(
"failed to reopen active file {}: {}".format(path, error))
+ writer.preresume_hook()
writer._do_queued_work()
return writer
+ def preresume_hook(self):
+ pass # Subclasses can override this as desired.
+
def check_dependencies(self):
for path, orig_stat in self._dependencies.items():
if not S_ISREG(orig_stat[ST_MODE]):
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index bb07be4..d67cf84 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -242,6 +242,9 @@ class ArvPutCollectionWriter(arvados.ResumableCollectionWriter):
else:
return writer
+ def preresume_hook(self):
+ print >>sys.stderr, "arv-put: Resuming previous upload. Bypass with the --no-resume option."
+
def checkpoint_state(self):
if self.cache is None:
return
commit 1a37f499f5b19fbbf508f5f9a0357a20d03d0f7e
Author: Brett Smith <brett at curoverse.com>
Date: Wed May 28 17:27:47 2014 -0400
2752: arv-put ResumeCache constructor only accepts path argument.
Refs #2752. Using exceptions to accept parsed arguments in the
constructor as well was too clever by half.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 496110d..bb07be4 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -160,11 +160,7 @@ class ResumeCache(object):
os.chmod(cls.CACHE_DIR, 0o700)
def __init__(self, file_spec):
- try:
- self.cache_file = open(file_spec, 'a+')
- except TypeError:
- file_spec = self.make_path(file_spec)
- self.cache_file = open(file_spec, 'a+')
+ self.cache_file = open(file_spec, 'a+')
self._lock_file(self.cache_file)
self.filename = self.cache_file.name
@@ -311,7 +307,7 @@ def main(arguments=None):
reporter = None
try:
- resume_cache = ResumeCache(args)
+ resume_cache = ResumeCache(ResumeCache.make_path(args))
if not args.resume:
resume_cache.restart()
except ResumeCacheConflict:
diff --git a/sdk/python/tests/test_arv-put.py b/sdk/python/tests/test_arv-put.py
index a7a3e89..4b7dc89 100644
--- a/sdk/python/tests/test_arv-put.py
+++ b/sdk/python/tests/test_arv-put.py
@@ -67,12 +67,12 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
def test_cache_names_ignore_irrelevant_arguments(self):
# Workaround: parse_arguments bails on --filename with a directory.
- args1 = arv_put.parse_arguments(['/tmp'])
- args2 = arv_put.parse_arguments(['/tmp'])
- args2.filename = 'tmp'
- self.assertEquals(arv_put.ResumeCache.make_path(args1),
- arv_put.ResumeCache.make_path(args2),
- "cache path considered --filename for directory")
+ path1 = self.cache_path_from_arglist(['/tmp'])
+ args = arv_put.parse_arguments(['/tmp'])
+ args.filename = 'tmp'
+ path2 = arv_put.ResumeCache.make_path(args)
+ self.assertEquals(path1, path2,
+ "cache path considered --filename for directory")
self.assertEquals(
self.cache_path_from_arglist(['-']),
self.cache_path_from_arglist(['-', '--max-manifest-depth', '1']),
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list