[ARVADOS] updated: 1.1.3-319-gc18fb83
Git user
git at public.curoverse.com
Tue Apr 3 12:51:24 EDT 2018
Summary of changes:
build/run-build-test-packages-one-target.sh | 1 +
build/run-library.sh | 48 +++++++++++---
doc/install/install-sso.html.textile.liquid | 6 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 76 ++++++++++++++--------
sdk/cwl/arvados_version.py | 20 ++++++
sdk/cwl/gittaggers.py | 6 +-
sdk/cwl/setup.py | 6 +-
sdk/cwl/tests/arvados-tests.yml | 25 +++++++
sdk/cwl/tests/wf/runin-reqs-wf.cwl | 58 +++++++++++++++++
sdk/cwl/tests/wf/runin-reqs-wf2.cwl | 59 +++++++++++++++++
sdk/cwl/tests/wf/runin-reqs-wf3.cwl | 59 +++++++++++++++++
sdk/cwl/tests/wf/runin-reqs-wf4.cwl | 59 +++++++++++++++++
sdk/python/arvados_version.py | 20 ++++++
sdk/python/gittaggers.py | 7 +-
sdk/python/setup.py | 6 +-
services/api/app/models/arvados_model.rb | 6 +-
services/api/app/models/collection.rb | 4 +-
services/api/app/models/container.rb | 34 ++++++----
services/api/app/models/container_request.rb | 5 +-
services/api/lib/arvados_model_updates.rb | 21 ++++++
services/api/lib/current_api_client.rb | 6 --
services/api/lib/sweep_trashed_collections.rb | 41 ------------
services/api/lib/sweep_trashed_objects.rb | 74 +++++++++++++++++++++
services/api/test/fixtures/groups.yml | 12 +++-
services/api/test/fixtures/jobs.yml | 11 ++++
services/api/test/unit/collection_test.rb | 14 ++--
services/api/test/unit/container_request_test.rb | 6 ++
services/api/test/unit/group_test.rb | 55 ++++++++++++++--
services/dockercleaner/arvados_version.py | 20 ++++++
services/dockercleaner/setup.py | 6 +-
services/fuse/arvados_version.py | 20 ++++++
services/fuse/setup.py | 8 +--
services/nodemanager/arvados_version.py | 20 ++++++
.../arvnodeman/computenode/driver/__init__.py | 6 +-
services/nodemanager/arvnodeman/daemon.py | 6 ++
services/nodemanager/arvnodeman/status.py | 4 +-
services/nodemanager/setup.py | 7 +-
.../nodemanager/tests/test_computenode_driver.py | 12 ++--
services/nodemanager/tests/test_daemon.py | 22 +++++++
services/nodemanager/tests/test_status.py | 3 +-
tools/crunchstat-summary/arvados_version.py | 20 ++++++
tools/crunchstat-summary/setup.py | 7 +-
42 files changed, 755 insertions(+), 151 deletions(-)
create mode 100644 sdk/cwl/arvados_version.py
create mode 100644 sdk/cwl/tests/wf/runin-reqs-wf.cwl
create mode 100644 sdk/cwl/tests/wf/runin-reqs-wf2.cwl
create mode 100644 sdk/cwl/tests/wf/runin-reqs-wf3.cwl
create mode 100644 sdk/cwl/tests/wf/runin-reqs-wf4.cwl
create mode 100644 sdk/python/arvados_version.py
create mode 100644 services/api/lib/arvados_model_updates.rb
delete mode 100644 services/api/lib/sweep_trashed_collections.rb
create mode 100644 services/api/lib/sweep_trashed_objects.rb
create mode 100644 services/dockercleaner/arvados_version.py
create mode 100644 services/fuse/arvados_version.py
create mode 100644 services/nodemanager/arvados_version.py
create mode 100644 tools/crunchstat-summary/arvados_version.py
via c18fb83a30a05cb96eb1e03dccf8e09fe570a4d4 (commit)
via 36670767328eddfebb17497797a8272aa461da51 (commit)
via e92e650a58dd5e27741212516af453d696055ccc (commit)
via 001a60dff02545c2d2476a437b1846c9ae633941 (commit)
via d0dd4abf2e364ad94c3bb8ad227faee28edda153 (commit)
via 5f463235cefa6a0c566ff3f0d71f09e0d3be2fce (commit)
via aebdddd8d1449ff7f8a6ba63054276e798dd79a0 (commit)
via a3da6f03cfb1bd6b36fdb1a51efeafcd7d64157e (commit)
via 8680023bb3804d3bd1f2dcac7d1a86ff14053fca (commit)
via fea63ac8216b2a15101bc75cd02586d934897160 (commit)
via 0381484cce9877c31231f92daa74ae971e43ba2f (commit)
via e9b4a6fc35b469794d0f6a080254006187501f66 (commit)
via 595606c78208fe8a783299091108b48ebccfc1c7 (commit)
via 2dbd8ea3bc22f8f0eecf4bbe752d0bec8bf88726 (commit)
via d506c0c47b3bee53d89fcbb9d3b933d4b3aee39a (commit)
via 24f9bc348608393ec88d5632e3b4a627420dcba1 (commit)
via 758a38ee0a652f2e8ca8bee94f0e182767bf499b (commit)
via dcb35293cb12d0fbcae5fd3f95026b3dc7713ab0 (commit)
via e64ab2bb55a6c547956c83907e72dda349cdc5e9 (commit)
via 37cfe352d0522015759b66ab552a964d74a76914 (commit)
via 3ada9e786796ae8861c375a1cc389e2a8873cd45 (commit)
via 73d6064c3816d2d569ae47cd5609e5dfdfe235a1 (commit)
via e8a24e57d286b4db22903762ebd888dc63d32206 (commit)
via 1a6777657c96a1a23678d9ec41e7d02ace55e5e8 (commit)
via 7a38d6b19120b3284533196ff3c94c2043826aef (commit)
via ab052248e5637d7d0a3ad8636dfd1b674a2ccc45 (commit)
via 0d5d9ba7fecfb5842930f27d9c450b33a8b4f153 (commit)
via c6ad0a2440f5e8f8361eeed07d0048c3d38c651b (commit)
via 0cbae7c90cb9298d930e8a0f001764df729fec86 (commit)
via 2f6a83dba97c5dbcbb3b077deafaa08797293edb (commit)
via 56b434f7e4ca8ceae70b4b75ce4e01f254fcd550 (commit)
via acecc56abbd717bd05bf4a6544a677f4b5a8fc9b (commit)
via 8e66062845cb567092de48edf9b2faaf1a4e2ae3 (commit)
via 0f2d5848629c68d8f59f140d4879d7172536dcb8 (commit)
via 5bac1f802ff40c4e82542a4fac49086fcc86e135 (commit)
via d73d6a3db684ad46eecd98fa8208594142ebdc2c (commit)
via 1c2372c62c11f6de9d452483e79dc15aaecc0578 (commit)
via fd8c6aa9f8e8588fca348d8ae5b9422dfb0ab7fb (commit)
via c9283683ff2e2dc3c52b4bc48b1d13d8dd1c9674 (commit)
via 9ca97ebf3f28fb88559a994daf222f629ad9d0aa (commit)
via fdaaf4f5b58406b10167714b57affe0905e3216b (commit)
via 04319cbe8d3e8c858bb6d93037e09d3af4271c42 (commit)
via 4998ea2c5a8b2bf9066de8c454d5fb990ff9cba6 (commit)
via d41ae3025fe6a06317aa8806bdc1913d994b4014 (commit)
via 71d59d43760dcacb0e6f56ec77f693b93f9e875d (commit)
via 0192a6a19302bbe469264319429c5f50322b1233 (commit)
via 5667012a090b6c561cdf0f84554d4673b2020954 (commit)
via 198a3409881f5b9f1fe77ae551efde1968bfb639 (commit)
via 45d780e20c41181c33562f99262bcf1f507f5a6a (commit)
via 368191c0f0f7c094a6ba7619dcf5e572a3f84f79 (commit)
via 54ef4b36d1fa48960a87f527c5dd69101486d908 (commit)
via 2b1a65816ab15bac35f260aa65f979d17e37323e (commit)
via a2e3be11c517402346702f8d4e3d2e851fec7b46 (commit)
via 1ea42486b1badd6ebd1a0071f3dd7bc1ec41b920 (commit)
via ac74647ec063c81654f0d4ec368a3e6bd74484a6 (commit)
via 2e1892bbd21d7ac8432c923fa84eb8526cfae558 (commit)
via 48a362bb93d1e53e879a563da85aecd070987f30 (commit)
via af2ec135f1b634e67205db1752d5ad65aa9c87b9 (commit)
via 9a92b8605be087ea76f2db614ea62173578948d9 (commit)
via 2baac8a6702551064b065f752f75a9b40dbec0f5 (commit)
via 378eeb13b1c4d188d07bde0abfe3955ad53d4beb (commit)
via 98fb00994467e805f764d799e78420ecac9c0879 (commit)
via 7a104e46ce7ebe905500824affa15c380fc5f2c5 (commit)
from a02012dd96d06fe438bc198b104a65eef1d4915a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c18fb83a30a05cb96eb1e03dccf8e09fe570a4d4
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Tue Apr 3 13:50:35 2018 -0300
12085: When an idle node disappears from the cloud node list, clear its counter.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/daemon.py b/services/nodemanager/arvnodeman/daemon.py
index 5f15b28..3147de3 100644
--- a/services/nodemanager/arvnodeman/daemon.py
+++ b/services/nodemanager/arvnodeman/daemon.py
@@ -219,6 +219,11 @@ class NodeManagerDaemonActor(actor_class):
# actor if necessary and forget about the node.
if record.actor:
try:
+ # If it's paired and idle, stop its idle time counter
+ # before removing the monitor actor.
+ if record.actor.get_state().get() == 'idle':
+ status.tracker.idle_out(
+ record.actor.arvados_node.get()['hostname'])
record.actor.stop()
except pykka.ActorDeadError:
pass
diff --git a/services/nodemanager/tests/test_daemon.py b/services/nodemanager/tests/test_daemon.py
index c497012..8050e69 100644
--- a/services/nodemanager/tests/test_daemon.py
+++ b/services/nodemanager/tests/test_daemon.py
@@ -665,6 +665,27 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
self.daemon.update_cloud_nodes([]).get(self.TIMEOUT)
self.busywait(lambda: 1 == self.last_shutdown.stop.call_count)
+ def test_idle_node_disappearing_clears_status_idle_time_counter(self):
+ size = testutil.MockSize(1)
+ status.tracker._idle_nodes = {}
+ cloud_nodes = [testutil.cloud_node_mock(1, size=size)]
+ arv_nodes = [testutil.arvados_node_mock(1, job_uuid=None)]
+ self.make_daemon(cloud_nodes, arv_nodes, [size])
+ self.busywait(lambda: 1 == self.paired_monitor_count())
+ for mon_ref in self.monitor_list():
+ monitor = mon_ref.proxy()
+ if monitor.cloud_node.get(self.TIMEOUT) is cloud_nodes[-1]:
+ break
+ else:
+ self.fail("monitor for idle node not found")
+ self.assertEqual(1, status.tracker.get('nodes_idle'))
+ hostname = monitor.arvados_node.get()['hostname']
+ self.assertIn(hostname, status.tracker._idle_nodes)
+ # Simulate the node disappearing from the cloud node list
+ self.daemon.update_cloud_nodes([]).get(self.TIMEOUT)
+ self.busywait(lambda: 0 == self.alive_monitor_count())
+ self.assertNotIn(hostname, status.tracker._idle_nodes)
+
def test_shutdown_actor_cleanup_copes_with_dead_actors(self):
self.make_daemon(cloud_nodes=[testutil.cloud_node_mock()])
self.assertEqual(1, self.alive_monitor_count())
commit 36670767328eddfebb17497797a8272aa461da51
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Apr 2 18:04:08 2018 -0300
12085: Add node_quota status & count different cloud errors separately.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/computenode/driver/__init__.py b/services/nodemanager/arvnodeman/computenode/driver/__init__.py
index fb8db51..7ed7435 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/__init__.py
@@ -127,7 +127,7 @@ class BaseComputeNodeDriver(RetryMixin):
try:
return self.real.list_nodes(**l)
except CLOUD_ERRORS:
- tracker.counter_add('cloud_errors')
+ tracker.counter_add('list_nodes_errors')
raise
def create_cloud_name(self, arvados_node):
@@ -186,7 +186,7 @@ class BaseComputeNodeDriver(RetryMixin):
try:
return self.search_for_now(kwargs['name'], 'list_nodes', self._name_key)
except ValueError:
- tracker.counter_add('cloud_errors')
+ tracker.counter_add('create_node_errors')
raise create_error
def post_create_node(self, cloud_node):
@@ -229,7 +229,7 @@ class BaseComputeNodeDriver(RetryMixin):
# it, which means destroy_node actually succeeded.
return True
# The node is still on the list. Re-raise.
- tracker.counter_add('cloud_errors')
+ tracker.counter_add('destroy_node_errors')
raise
# Now that we've defined all our own methods, delegate generic, public
diff --git a/services/nodemanager/arvnodeman/daemon.py b/services/nodemanager/arvnodeman/daemon.py
index a6e73e2..5f15b28 100644
--- a/services/nodemanager/arvnodeman/daemon.py
+++ b/services/nodemanager/arvnodeman/daemon.py
@@ -270,6 +270,7 @@ class NodeManagerDaemonActor(actor_class):
updates.setdefault('nodes_'+s, 0)
updates['nodes_'+s] += 1
updates['nodes_wish'] = len(self.last_wishlist)
+ updates['node_quota'] = self.node_quota
status.tracker.update(updates)
def _state_counts(self, size):
diff --git a/services/nodemanager/arvnodeman/status.py b/services/nodemanager/arvnodeman/status.py
index 20e30e1..1e18996 100644
--- a/services/nodemanager/arvnodeman/status.py
+++ b/services/nodemanager/arvnodeman/status.py
@@ -78,7 +78,9 @@ class Tracker(object):
def __init__(self):
self._mtx = threading.Lock()
self._latest = {
- 'cloud_errors': 0,
+ 'list_nodes_errors': 0,
+ 'create_node_errors': 0,
+ 'destroy_node_errors': 0,
'boot_failures': 0,
'actor_exceptions': 0
}
diff --git a/services/nodemanager/tests/test_computenode_driver.py b/services/nodemanager/tests/test_computenode_driver.py
index b191330..128a29e 100644
--- a/services/nodemanager/tests/test_computenode_driver.py
+++ b/services/nodemanager/tests/test_computenode_driver.py
@@ -72,7 +72,7 @@ class ComputeNodeDriverTestCase(unittest.TestCase):
def test_create_node_only_cloud_errors_are_counted(self):
- status.tracker.update({'cloud_errors': 0})
+ status.tracker.update({'create_node_errors': 0})
errors = [(config.CLOUD_ERRORS[0], True), (KeyError, False)]
self.driver_mock().list_images.return_value = []
driver = self.TestBaseComputeNodeDriver({}, {}, {}, self.driver_mock)
@@ -83,10 +83,10 @@ class ComputeNodeDriverTestCase(unittest.TestCase):
driver.create_node('1', 'id_1')
if is_cloud_error:
error_count += 1
- self.assertEqual(error_count, status.tracker.get('cloud_errors'))
+ self.assertEqual(error_count, status.tracker.get('create_node_errors'))
def test_list_nodes_only_cloud_errors_are_counted(self):
- status.tracker.update({'cloud_errors': 0})
+ status.tracker.update({'list_nodes_errors': 0})
errors = [(config.CLOUD_ERRORS[0], True), (KeyError, False)]
driver = self.TestBaseComputeNodeDriver({}, {}, {}, self.driver_mock)
error_count = 0
@@ -96,10 +96,10 @@ class ComputeNodeDriverTestCase(unittest.TestCase):
driver.list_nodes()
if is_cloud_error:
error_count += 1
- self.assertEqual(error_count, status.tracker.get('cloud_errors'))
+ self.assertEqual(error_count, status.tracker.get('list_nodes_errors'))
def test_destroy_node_only_cloud_errors_are_counted(self):
- status.tracker.update({'cloud_errors': 0})
+ status.tracker.update({'destroy_node_errors': 0})
errors = [(config.CLOUD_ERRORS[0], True), (KeyError, False)]
self.driver_mock().list_nodes.return_value = [testutil.MockSize(1)]
driver = self.TestBaseComputeNodeDriver({}, {}, {}, self.driver_mock)
@@ -110,4 +110,4 @@ class ComputeNodeDriverTestCase(unittest.TestCase):
driver.destroy_node(testutil.MockSize(1))
if is_cloud_error:
error_count += 1
- self.assertEqual(error_count, status.tracker.get('cloud_errors'))
+ self.assertEqual(error_count, status.tracker.get('destroy_node_errors'))
diff --git a/services/nodemanager/tests/test_daemon.py b/services/nodemanager/tests/test_daemon.py
index 4b2b2d4..c497012 100644
--- a/services/nodemanager/tests/test_daemon.py
+++ b/services/nodemanager/tests/test_daemon.py
@@ -143,6 +143,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin,
size = testutil.MockSize(1)
self.make_daemon(want_sizes=[size])
self.busywait(lambda: self.node_setup.start.called)
+ self.assertIn('node_quota', status.tracker._latest)
def check_monitors_arvados_nodes(self, *arv_nodes):
self.busywait(lambda: len(arv_nodes) == len(self.monitored_arvados_nodes()))
diff --git a/services/nodemanager/tests/test_status.py b/services/nodemanager/tests/test_status.py
index 2088a0d..2a1c0fc 100644
--- a/services/nodemanager/tests/test_status.py
+++ b/services/nodemanager/tests/test_status.py
@@ -65,7 +65,8 @@ class StatusServerUpdates(unittest.TestCase):
with TestServer() as srv:
resp = srv.get_status()
# Test counters existance
- for counter in ['cloud_errors', 'boot_failures', 'actor_exceptions']:
+ for counter in ['list_nodes_errors', 'create_node_errors',
+ 'destroy_node_errors', 'boot_failures', 'actor_exceptions']:
self.assertIn(counter, resp)
# Test counter increment
for count in range(1, 3):
commit e92e650a58dd5e27741212516af453d696055ccc
Merge: a02012d 001a60d
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Apr 2 17:01:24 2018 -0300
Merge branch 'master' into 12085-anm-metrics
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list