[ARVADOS] updated: 1.3.0-968-g43424cf94
Git user
git at public.curoverse.com
Tue Jun 4 12:36:00 UTC 2019
Summary of changes:
sdk/python/arvados/commands/put.py | 44 +++++++++++-----
sdk/python/tests/test_arv_put.py | 101 ++++++++++++++++++++++++++++++++-----
2 files changed, 119 insertions(+), 26 deletions(-)
via 43424cf94b198085e9cfa1a139e9a492b2e1dced (commit)
via 6024f7a6fa0a3518db5d2e982a53e83623f9b82b (commit)
via 8f39f0b74291009f036359bd651020e159a75d10 (commit)
via 081b65fb2c53f9faba40fedd2de9cb9f1a860016 (commit)
from d56c1ef84083152a6623d8661ca45b93a605e0ea (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 43424cf94b198085e9cfa1a139e9a492b2e1dced
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Tue Jun 4 09:34:53 2019 -0300
14930: Fixes utc offset calculation when naive datetime provided & adds tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index d13c2cd50..642472c8a 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -1119,12 +1119,12 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr,
else:
if trash_at.tzinfo is not None:
# Timezone aware datetime provided.
- delta = trash_at.utcoffset()
+ utcoffset = trash_at.utcoffset()
else:
# Timezone naive datetime provided. Assume is local.
- delta = datetime.datetime.now() - datetime.datetime.utcnow()
+ utcoffset = datetime.timedelta(hours=-time.timezone/3600)
# Convert to UTC timezone naive datetime.
- trash_at = trash_at.replace(tzinfo=None) - delta
+ trash_at = trash_at.replace(tzinfo=None) - utcoffset
if trash_at <= datetime.datetime.utcnow():
logger.error("--trash-at argument should be set in the future")
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index d26957c21..42ab24235 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -12,6 +12,7 @@ from builtins import str
from builtins import range
from functools import partial
import apiclient
+import ciso8601
import datetime
import hashlib
import json
@@ -212,7 +213,7 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
def test_cache_is_locked(self):
with tempfile.NamedTemporaryFile() as cachefile:
- arv_put.ResumeCache(cachefile.name)
+ _ = arv_put.ResumeCache(cachefile.name)
self.assertRaises(arv_put.ResumeCacheConflict,
arv_put.ResumeCache, cachefile.name)
@@ -1160,6 +1161,82 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
c = arv_put.api_client.collections().get(uuid=updated_col['uuid']).execute()
self.assertRegex(c['manifest_text'], r'^\..* .*:44:file2\n')
+ def test_put_collection_with_utc_expiring_datetime(self):
+ tmpdir = self.make_tmpdir()
+ trash_at = (datetime.datetime.utcnow() + datetime.timedelta(days=90)).strftime('%Y%m%dT%H%MZ')
+ with open(os.path.join(tmpdir, 'file1'), 'w') as f:
+ f.write('Relaxing in basins at the end of inlets terminates the endless tests from the box')
+ col = self.run_and_find_collection(
+ "",
+ ['--no-progress', '--trash-at', trash_at, tmpdir])
+ self.assertNotEqual(None, col['uuid'])
+ c = arv_put.api_client.collections().get(uuid=col['uuid']).execute()
+ self.assertEqual(ciso8601.parse_datetime(trash_at),
+ ciso8601.parse_datetime(c['trash_at']))
+
+ def test_put_collection_with_timezone_aware_expiring_datetime(self):
+ tmpdir = self.make_tmpdir()
+ trash_at = (datetime.datetime.utcnow() + datetime.timedelta(days=90)).strftime('%Y%m%dT%H%M-0300')
+ with open(os.path.join(tmpdir, 'file1'), 'w') as f:
+ f.write('Relaxing in basins at the end of inlets terminates the endless tests from the box')
+ col = self.run_and_find_collection(
+ "",
+ ['--no-progress', '--trash-at', trash_at, tmpdir])
+ self.assertNotEqual(None, col['uuid'])
+ c = arv_put.api_client.collections().get(uuid=col['uuid']).execute()
+ self.assertEqual(
+ ciso8601.parse_datetime(trash_at).replace(tzinfo=None)+datetime.timedelta(hours=3),
+ ciso8601.parse_datetime(c['trash_at']).replace(tzinfo=None))
+
+ def test_put_collection_with_timezone_naive_expiring_datetime(self):
+ tmpdir = self.make_tmpdir()
+ trash_at = (datetime.datetime.utcnow() + datetime.timedelta(days=90)).strftime('%Y%m%dT%H%M')
+ with open(os.path.join(tmpdir, 'file1'), 'w') as f:
+ f.write('Relaxing in basins at the end of inlets terminates the endless tests from the box')
+ col = self.run_and_find_collection(
+ "",
+ ['--no-progress', '--trash-at', trash_at, tmpdir])
+ self.assertNotEqual(None, col['uuid'])
+ c = arv_put.api_client.collections().get(uuid=col['uuid']).execute()
+ self.assertEqual(
+ ciso8601.parse_datetime(trash_at) - datetime.timedelta(hours=-time.timezone/3600),
+ ciso8601.parse_datetime(c['trash_at']).replace(tzinfo=None))
+
+ def test_put_collection_with_invalid_absolute_expiring_datetime(self):
+ tmpdir = self.make_tmpdir()
+ with open(os.path.join(tmpdir, 'file1'), 'w') as f:
+ f.write('Relaxing in basins at the end of inlets terminates the endless tests from the box')
+ with self.assertRaises(AssertionError):
+ self.run_and_find_collection(
+ "",
+ ['--no-progress', '--trash-at', 'tomorrow at noon', tmpdir])
+
+ def test_put_collection_with_relative_expiring_datetime(self):
+ expire_after = 7
+ dt_before = datetime.datetime.utcnow() + datetime.timedelta(days=expire_after)
+ tmpdir = self.make_tmpdir()
+ with open(os.path.join(tmpdir, 'file1'), 'w') as f:
+ f.write('Relaxing in basins at the end of inlets terminates the endless tests from the box')
+ col = self.run_and_find_collection(
+ "",
+ ['--no-progress', '--trash-after', str(expire_after), tmpdir])
+ self.assertNotEqual(None, col['uuid'])
+ dt_after = datetime.datetime.utcnow() + datetime.timedelta(days=expire_after)
+ c = arv_put.api_client.collections().get(uuid=col['uuid']).execute()
+ trash_at = ciso8601.parse_datetime(c['trash_at']).replace(tzinfo=None)
+ self.assertTrue(dt_before < trash_at)
+ self.assertTrue(dt_after > trash_at)
+
+ def test_put_collection_with_invalid_relative_expiring_datetime(self):
+ expire_after = 0 # Should be >= 1
+ tmpdir = self.make_tmpdir()
+ with open(os.path.join(tmpdir, 'file1'), 'w') as f:
+ f.write('Relaxing in basins at the end of inlets terminates the endless tests from the box')
+ with self.assertRaises(AssertionError):
+ self.run_and_find_collection(
+ "",
+ ['--no-progress', '--trash-after', str(expire_after), tmpdir])
+
def test_upload_directory_reference_without_trailing_slash(self):
tmpdir1 = self.make_tmpdir()
tmpdir2 = self.make_tmpdir()
commit 6024f7a6fa0a3518db5d2e982a53e83623f9b82b
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Jun 3 17:42:07 2019 -0300
14930: Removes unused variable assignments on tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 540e06c6c..d26957c21 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -212,7 +212,7 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase):
def test_cache_is_locked(self):
with tempfile.NamedTemporaryFile() as cachefile:
- cache = arv_put.ResumeCache(cachefile.name)
+ arv_put.ResumeCache(cachefile.name)
self.assertRaises(arv_put.ResumeCacheConflict,
arv_put.ResumeCache, cachefile.name)
@@ -311,7 +311,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
def test_passing_nonexistant_path_raise_exception(self):
uuid_str = str(uuid.uuid4())
with self.assertRaises(arv_put.PathDoesNotExistError):
- cwriter = arv_put.ArvPutUploadJob(["/this/path/does/not/exist/{}".format(uuid_str)])
+ arv_put.ArvPutUploadJob(["/this/path/does/not/exist/{}".format(uuid_str)])
def test_writer_works_without_cache(self):
cwriter = arv_put.ArvPutUploadJob(['/dev/null'], resume=False)
@@ -843,7 +843,7 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers,
fake_httplib2_response(403), b'{}')
with mock.patch('arvados.collection.Collection.save_new',
new=coll_save_mock):
- with self.assertRaises(SystemExit) as exc_test:
+ with self.assertRaises(SystemExit):
self.call_main_with_args(['/dev/null'])
self.assertRegex(
self.main_stderr.getvalue(), matcher)
@@ -918,7 +918,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
BAD_UUID = 'zzzzz-tpzed-zzzzzzzzzzzzzzz'
self.authorize_with('active')
with self.assertRaises(apiclient.errors.HttpError):
- result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
+ arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID,
0)
def test_short_put_from_stdin(self):
@@ -979,7 +979,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
# we're about to create is not present in our test fixture.
manifest_uuid = "00b4e9f40ac4dd432ef89749f1c01e74+47"
with self.assertRaises(apiclient.errors.HttpError):
- notfound = arv_put.api_client.collections().get(
+ arv_put.api_client.collections().get(
uuid=manifest_uuid).execute()
datadir = self.make_tmpdir()
@@ -990,7 +990,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.ENVIRON)
- (out, err) = p.communicate()
+ (_, err) = p.communicate()
self.assertRegex(err.decode(), r'INFO: Collection saved as ')
self.assertEqual(p.returncode, 0)
@@ -1030,7 +1030,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.ENVIRON)
- (out, err) = p.communicate()
+ (_, err) = p.communicate()
self.assertRegex(err.decode(), r'INFO: Creating new cache file at ')
self.assertEqual(p.returncode, 0)
cache_filepath = re.search(r'INFO: Creating new cache file at (.*)',
@@ -1053,7 +1053,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.ENVIRON)
- (out, err) = p.communicate()
+ (_, err) = p.communicate()
self.assertRegex(
err.decode(),
r'INFO: Cache expired, starting from scratch.*')
@@ -1069,7 +1069,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.ENVIRON)
- (out, err) = p.communicate()
+ (_, err) = p.communicate()
self.assertRegex(err.decode(), r'INFO: Creating new cache file at ')
self.assertEqual(p.returncode, 0)
cache_filepath = re.search(r'INFO: Creating new cache file at (.*)',
@@ -1091,7 +1091,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.ENVIRON)
- (out, err) = p.communicate()
+ (_, err) = p.communicate()
self.assertRegex(
err.decode(),
r'ERROR: arv-put: Resume cache contains invalid signature.*')
@@ -1111,7 +1111,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.ENVIRON)
- (out, err) = p.communicate()
+ (_, err) = p.communicate()
self.assertRegex(err.decode(), r'INFO: Creating new cache file at ')
self.assertEqual(p.returncode, 0)
cache_filepath = re.search(r'INFO: Creating new cache file at (.*)',
@@ -1135,7 +1135,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.ENVIRON)
- (out, err) = p.communicate()
+ (_, err) = p.communicate()
self.assertRegex(
err.decode(),
r'WARNING: Uploaded file \'.*barfile.txt\' access token expired, will re-upload it from scratch')
commit 8f39f0b74291009f036359bd651020e159a75d10
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Jun 3 16:28:06 2019 -0300
14930: Assumes timezone-less provided trash datetimes as local.
Depending of the local system's clock setting, it makes the necessary
correction to express it as UTC.
Also, provides examples on the command's help output.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index a21719cab..d13c2cd50 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -239,7 +239,8 @@ _group = upload_opts.add_mutually_exclusive_group()
_group.add_argument('--trash-at', metavar='YYYY-MM-DD HH:MM', default=None,
help="""
Set the trash date of the resulting collection to an absolute date in the future.
-The accepted format is defined by the ISO 8601 standard.
+The accepted format is defined by the ISO 8601 standard. Examples: 20090103, 2009-01-03, 20090103T181505, 2009-01-03T18:15:05.\n
+Timezone information can be added. If not, the provided date/time is assumed as being in the local system's timezone.
""")
_group.add_argument('--trash-after', type=int, metavar='DAYS', default=None,
help="""
@@ -486,8 +487,11 @@ class ArvPutUploadJob(object):
self.exclude_names = exclude_names
self._trash_at = trash_at
- if self._trash_at is not None and type(self._trash_at) not in [datetime.datetime, datetime.timedelta]:
- raise TypeError('trash_at should be None, datetime or timedelta')
+ if self._trash_at is not None:
+ if type(self._trash_at) not in [datetime.datetime, datetime.timedelta]:
+ raise TypeError('trash_at should be None, timezone-naive datetime or timedelta')
+ if type(self._trash_at) == datetime.datetime and self._trash_at.tzinfo is not None:
+ raise TypeError('provided trash_at datetime should be timezone-naive')
if not self.use_cache and self.resume:
raise ArvPutArgumentConflict('resume cannot be True when use_cache is False')
@@ -1110,13 +1114,18 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr,
try:
trash_at = ciso8601.parse_datetime(args.trash_at)
except:
- logger.error("--trash-at argument format invalid, should be YYYY-MM-DDTHH:MM.")
+ logger.error("--trash-at argument format invalid, use --help to see examples.")
sys.exit(1)
else:
if trash_at.tzinfo is not None:
- # Timezone-aware datetime provided, convert to non-aware UTC
- delta = trash_at.tzinfo.utcoffset(None)
- trash_at = trash_at.replace(tzinfo=None) - delta
+ # Timezone aware datetime provided.
+ delta = trash_at.utcoffset()
+ else:
+ # Timezone naive datetime provided. Assume is local.
+ delta = datetime.datetime.now() - datetime.datetime.utcnow()
+ # Convert to UTC timezone naive datetime.
+ trash_at = trash_at.replace(tzinfo=None) - delta
+
if trash_at <= datetime.datetime.utcnow():
logger.error("--trash-at argument should be set in the future")
sys.exit(1)
commit 081b65fb2c53f9faba40fedd2de9cb9f1a860016
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Jun 3 14:05:46 2019 -0300
14930: Moves the trash_at compute logic into its own method.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index aa8467cb9..a21719cab 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -484,9 +484,9 @@ class ArvPutUploadJob(object):
self.follow_links = follow_links
self.exclude_paths = exclude_paths
self.exclude_names = exclude_names
- self.trash_at = trash_at
+ self._trash_at = trash_at
- if self.trash_at is not None and type(self.trash_at) not in [datetime.datetime, datetime.timedelta]:
+ if self._trash_at is not None and type(self._trash_at) not in [datetime.datetime, datetime.timedelta]:
raise TypeError('trash_at should be None, datetime or timedelta')
if not self.use_cache and self.resume:
@@ -628,11 +628,18 @@ class ArvPutUploadJob(object):
if self.use_cache:
self._cache_file.close()
- def save_collection(self):
- if type(self.trash_at) == datetime.timedelta:
- # Get an absolute datetime for trash_at before saving.
- self.trash_at = datetime.datetime.utcnow() + self.trash_at
+ def _collection_trash_at(self):
+ """
+ Returns the trash date that the collection should use at save time.
+ Takes into account absolute/relative trash_at values requested
+ by the user.
+ """
+ if type(self._trash_at) == datetime.timedelta:
+ # Get an absolute datetime for trash_at
+ return datetime.datetime.utcnow() + self._trash_at
+ return self._trash_at
+ def save_collection(self):
if self.update:
# Check if files should be updated on the remote collection.
for fp in self._file_paths:
@@ -648,7 +655,7 @@ class ArvPutUploadJob(object):
pass
self._remote_collection.save(storage_classes=self.storage_classes,
num_retries=self.num_retries,
- trash_at=self.trash_at)
+ trash_at=self._collection_trash_at())
else:
if self.storage_classes is None:
self.storage_classes = ['default']
@@ -657,7 +664,7 @@ class ArvPutUploadJob(object):
storage_classes=self.storage_classes,
ensure_unique_name=self.ensure_unique_name,
num_retries=self.num_retries,
- trash_at=self.trash_at)
+ trash_at=self._collection_trash_at())
def destroy_cache(self):
if self.use_cache:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list