[ARVADOS] updated: ed16f0c1f600acd7bab81796ec22fde90ecdfa7f
Git user
git at public.curoverse.com
Fri Sep 16 16:26:50 EDT 2016
Summary of changes:
.../controllers/container_requests_controller.rb | 4 +-
.../app/controllers/work_units_controller.rb | 2 +-
apps/workbench/app/helpers/application_helper.rb | 27 ++-
.../views/container_requests/_show_inputs.html.erb | 9 +-
build/run-tests.sh | 20 +-
sdk/cwl/arvados_cwl/__init__.py | 21 +-
sdk/cwl/arvados_cwl/arvjob.py | 86 ++++----
sdk/cwl/arvados_cwl/arvworkflow.py | 41 ++++
sdk/cwl/arvados_cwl/perf.py | 15 ++
sdk/cwl/arvados_cwl/runner.py | 129 +++++++-----
sdk/cwl/tests/test_container.py | 1 +
sdk/cwl/tests/test_job.py | 1 +
sdk/cwl/tests/test_submit.py | 46 +++-
sdk/cwl/tests/wf/expect_packed.cwl | 26 +++
services/api/app/models/arvados_model.rb | 12 ++
services/api/app/models/container.rb | 43 ++++
services/api/app/models/container_request.rb | 22 +-
services/api/app/models/job.rb | 12 --
services/api/test/fixtures/container_requests.yml | 5 +-
services/api/test/fixtures/containers.yml | 4 -
services/api/test/fixtures/workflows.yml | 2 +-
services/api/test/unit/container_request_test.rb | 39 +++-
services/api/test/unit/container_test.rb | 231 +++++++++++++++++++++
services/fuse/tests/integration_test.py | 52 +++--
services/fuse/tests/mount_test_base.py | 26 ++-
25 files changed, 703 insertions(+), 173 deletions(-)
create mode 100644 sdk/cwl/arvados_cwl/arvworkflow.py
create mode 100644 sdk/cwl/arvados_cwl/perf.py
create mode 100644 sdk/cwl/tests/wf/expect_packed.cwl
discards 63228003517d7b30fb4d14f24cc7c243dd33ed24 (commit)
discards f5d65486546b508fe2b055b37a512f14c4a5498e (commit)
via ed16f0c1f600acd7bab81796ec22fde90ecdfa7f (commit)
via 17f2b714df870dffc6320173d6a2348243ff7992 (commit)
via 3bf898db1a6f0db043060cd601131b17bd6ef82d (commit)
via d9ef9d79b7a81a20d942efdf7af66fe5351eeec4 (commit)
via 275637f3ce7fcaacd8f41cd038b15332785298ef (commit)
via 920a9b20de934454768a43ce6d8a8d8ff4ca0840 (commit)
via cae94f22b760c6c6899fc4d23db15d389535ff0a (commit)
via 6430b728cbce69b56adc01432fc9351088724efd (commit)
via dfda2d511b5581968c5fa456c78776ac453dedd9 (commit)
via 503c686bc80825d00980a970af69ec60f9e6ce9b (commit)
via 2dc308e3106090e2a19185e26cff22c6136bb5c4 (commit)
via 1c582d3ee6df9e3ad114ec3ec0cbc58f52626ffb (commit)
via 7382b7a2d35e93944ff364d36017f51e9a1cd392 (commit)
via caa4cce0bc97ae99bc1657c3126d3c098e959eaa (commit)
via 89c7264f5a3b53043762fb8ced7bdb997ed2a120 (commit)
via fcdac0fbe5daa25a8206ac8495ec43b05bb48446 (commit)
via 3c69428a8165f485140f1bbbbb899a308e289df2 (commit)
via 6284bed37f6b2bb3c0671b401aa0d99d5ef7a5b7 (commit)
via 849f35d1cafa1b58e55832426436689b95d69ac5 (commit)
via eb097927c63ac10900489b7728efcb639dd286b1 (commit)
via a02b4f41cb495ecfea49ac0da1f6da0aec385f7d (commit)
via 6139a7b24e650aadd16857ed4383cfe4363ea45b (commit)
via 93e448118cfb620196967d05e8914fadd68b9d67 (commit)
via 1a87c77a5cfe200921a9d09e995383bc675e7370 (commit)
via a47f0ade77ea89419f23b7746f51d49d24576cf3 (commit)
via 1d0bae2cf9ab3c493dc974cd06262a657eb608fb (commit)
via 62121f9b2f15cd35a6ce5b6e66ccaaae1c1af834 (commit)
via f137e22a65cc29d00146a4735fe54d72b299984b (commit)
via e33cb63676097f15842cefe8c3a750965daca751 (commit)
via bcd205719cb6dcaacfae9ee99b43d8e35fe8f072 (commit)
via de4cfeaab629e99014deb10792637649511c91d0 (commit)
via f72506f1056aa4e867c6a0ff68a815bd55244536 (commit)
via 1b4bfe1a002732b1e9246be8b2679fec0a262202 (commit)
via 2b50a5838d80dc6e93864a0b8180b423497a8e1d (commit)
via 4eef0a298418a751b5941fd6f4bf32d91b817d54 (commit)
via 8100ee8f50d0c8b0340640db10745e44c0f4571b (commit)
via 374a802e502d044973fd21ca68d2f6ab707bd770 (commit)
via 40196e450ef489a31eeabf2c11a3969094e185b2 (commit)
via d42022147d700c5c0363a4429ceb04567fcc0b3b (commit)
via 0b187b107128a57c1f0c00057d39979ab53d4137 (commit)
via 014e73c96c56bc588a02bfdbdd653c10be30f80c (commit)
via 540adbf18728558df8797bf176ec666061cea934 (commit)
via 0f53d219c91812baeac4cff3387196c6501ec0c0 (commit)
via 86af9b2ebc9240e9041870786e87021c94f11eea (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (63228003517d7b30fb4d14f24cc7c243dd33ed24)
\
N -- N -- N (ed16f0c1f600acd7bab81796ec22fde90ecdfa7f)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ed16f0c1f600acd7bab81796ec22fde90ecdfa7f
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 14 10:19:21 2016 -0400
10008: Use mocks to ensure collection is re-fetched.
diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index b485037..c79b131 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -61,7 +61,6 @@ class IntegrationTest(unittest.TestCase):
def setUp(self):
self.mnt = tempfile.mkdtemp()
run_test_server.authorize_with('active')
- self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
def tearDown(self):
os.rmdir(self.mnt)
diff --git a/services/fuse/tests/test_token_expiry.py b/services/fuse/tests/test_token_expiry.py
index 790c819..1b1be18 100644
--- a/services/fuse/tests/test_token_expiry.py
+++ b/services/fuse/tests/test_token_expiry.py
@@ -1,9 +1,12 @@
-import arvados
import apiclient
+import arvados
+import arvados_fuse
import logging
import mock
+import multiprocessing
import os
import re
+import sys
import time
import unittest
@@ -12,33 +15,46 @@ from .integration_test import IntegrationTest
logger = logging.getLogger('arvados.arv-mount')
class TokenExpiryTest(IntegrationTest):
+ def __init__(self, *args, **kwargs):
+ super(TokenExpiryTest, self).__init__(*args, **kwargs)
+ self.test_start_time = time.time()
+ self.time_now = int(time.time())+1
+
+ def fake_time(self):
+ self.time_now += 1
+ return self.time_now
+
+ orig_open = arvados_fuse.Operations.open
+ def fake_open(self, operations, *args, **kwargs):
+ self.time_now += 86400*13
+ logger.debug('opening file at time=%f', self.time_now)
+ return self.orig_open(operations, *args, **kwargs)
+
+ @mock.patch.object(arvados_fuse.Operations, 'open', autospec=True)
+ @mock.patch('time.time')
@mock.patch('arvados.keep.KeepClient.get')
@IntegrationTest.mount(argv=['--mount-by-id', 'zzz'])
- def test_refresh_old_manifest(self, mocked_get):
+ def test_refresh_old_manifest(self, mocked_get, mocked_time, mocked_open):
mocked_get.return_value = 'fake data'
+ mocked_time.side_effect = self.fake_time
+ mocked_open.side_effect = self.fake_open
+
+ with mock.patch.object(self.mount.api, 'collections', wraps=self.mount.api.collections) as mocked_collections:
+ mocked_collections.return_value = mocked_collections()
+ with mock.patch.object(self.mount.api.collections(), 'get', wraps=self.mount.api.collections().get) as mocked_get:
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
- self.api._rootDesc["blobSignatureTtl"] = 2
- old_exp = int(time.time()) + 86400*14
- self.pool_test(os.path.join(self.mnt, 'zzz'))
- want_exp = int(time.time()) + 86400*14
-
- got_loc = mocked_get.call_args[0][0]
- got_exp = int(
- re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1),
- 16)
- self.assertGreaterEqual(
- got_exp, want_exp-2,
- msg='now+2w = {:x}, but fuse fetched locator {} (old_exp {:x})'.format(
- want_exp, got_loc, old_exp))
- self.assertLessEqual(
- got_exp, want_exp,
- msg='server is not using the expected 2w TTL; test is ineffective')
+ self.assertEqual(3, mocked_open.call_count)
+ self.assertEqual(
+ 4, mocked_get.call_count,
+ 'Not enough calls to collections().get(): expected 4, got {!r}'.format(
+ mocked_get.mock_calls))
@staticmethod
def _test_refresh_old_manifest(self, zzz):
uuid = 'zzzzz-4zz18-op4e2lbej01tcvu'
fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt'
os.listdir(os.path.join(zzz, uuid))
- time.sleep(3)
- with open(os.path.join(zzz, uuid, fnm)) as f:
- f.read()
+ for _ in range(3):
+ with open(os.path.join(zzz, uuid, fnm)) as f:
+ f.read()
commit 17f2b714df870dffc6320173d6a2348243ff7992
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Sep 8 02:10:34 2016 -0400
10008: Move token exp test back to IntegrationTest
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 39f1109..2aba2e7 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -1128,36 +1128,3 @@ class FuseMagicTestPDHOnly(MountTestBase):
def test_with_default_by_id(self):
self.verify_pdh_only(skip_pdh_only=True)
-
-def _test_refresh_old_manifest(zzz):
- fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt'
- os.listdir(os.path.join(zzz))
- time.sleep(3)
- with open(os.path.join(zzz, fnm)) as f:
- f.read()
-
-class TokenExpiryTest(MountTestBase):
- def setUp(self):
- super(TokenExpiryTest, self).setUp(local_store=False)
-
- @mock.patch('arvados.keep.KeepClient.get')
- def runTest(self, mocked_get):
- self.api._rootDesc = {"blobSignatureTtl": 2}
- mnt = self.make_mount(fuse.CollectionDirectory, collection_record='zzzzz-4zz18-op4e2lbej01tcvu')
- mocked_get.return_value = 'fake data'
-
- old_exp = int(time.time()) + 86400*14
- self.pool.apply(_test_refresh_old_manifest, (self.mounttmp,))
- want_exp = int(time.time()) + 86400*14
-
- got_loc = mocked_get.call_args[0][0]
- got_exp = int(
- re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1),
- 16)
- self.assertGreaterEqual(
- got_exp, want_exp-2,
- msg='now+2w = {:x}, but fuse fetched locator {} (old_exp {:x})'.format(
- want_exp, got_loc, old_exp))
- self.assertLessEqual(
- got_exp, want_exp,
- msg='server is not using the expected 2w TTL; test is ineffective')
diff --git a/services/fuse/tests/test_token_expiry.py b/services/fuse/tests/test_token_expiry.py
new file mode 100644
index 0000000..790c819
--- /dev/null
+++ b/services/fuse/tests/test_token_expiry.py
@@ -0,0 +1,44 @@
+import arvados
+import apiclient
+import logging
+import mock
+import os
+import re
+import time
+import unittest
+
+from .integration_test import IntegrationTest
+
+logger = logging.getLogger('arvados.arv-mount')
+
+class TokenExpiryTest(IntegrationTest):
+ @mock.patch('arvados.keep.KeepClient.get')
+ @IntegrationTest.mount(argv=['--mount-by-id', 'zzz'])
+ def test_refresh_old_manifest(self, mocked_get):
+ mocked_get.return_value = 'fake data'
+
+ self.api._rootDesc["blobSignatureTtl"] = 2
+ old_exp = int(time.time()) + 86400*14
+ self.pool_test(os.path.join(self.mnt, 'zzz'))
+ want_exp = int(time.time()) + 86400*14
+
+ got_loc = mocked_get.call_args[0][0]
+ got_exp = int(
+ re.search(r'\+A[0-9a-f]+@([0-9a-f]+)', got_loc).group(1),
+ 16)
+ self.assertGreaterEqual(
+ got_exp, want_exp-2,
+ msg='now+2w = {:x}, but fuse fetched locator {} (old_exp {:x})'.format(
+ want_exp, got_loc, old_exp))
+ self.assertLessEqual(
+ got_exp, want_exp,
+ msg='server is not using the expected 2w TTL; test is ineffective')
+
+ @staticmethod
+ def _test_refresh_old_manifest(self, zzz):
+ uuid = 'zzzzz-4zz18-op4e2lbej01tcvu'
+ fnm = 'zzzzz-8i9sb-0vsrcqi7whchuil.log.txt'
+ os.listdir(os.path.join(zzz, uuid))
+ time.sleep(3)
+ with open(os.path.join(zzz, uuid, fnm)) as f:
+ f.read()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list