[ARVADOS] updated: 743341f8a70b126b82ef3293bd9f6c2bc47ec29a
git at public.curoverse.com
git at public.curoverse.com
Fri Nov 20 15:32:18 EST 2015
Summary of changes:
sdk/cli/bin/crunch-job | 30 ++++--
sdk/python/arvados/events.py | 25 ++---
sdk/python/arvados/keep.py | 117 +++++++++++++++------
sdk/python/tests/test_keep_client.py | 7 ++
services/fuse/arvados_fuse/__init__.py | 20 +++-
services/fuse/bin/arv-mount | 67 ++++++++++++
services/fuse/setup.py | 2 +-
services/fuse/tests/test_mount.py | 4 +
.../arvnodeman/computenode/driver/__init__.py | 2 +
.../arvnodeman/computenode/driver/azure.py | 8 ++
.../arvnodeman/computenode/driver/dummy.py | 1 +
.../arvnodeman/computenode/driver/ec2.py | 8 ++
services/nodemanager/arvnodeman/daemon.py | 26 +++--
services/nodemanager/arvnodeman/jobqueue.py | 12 ++-
14 files changed, 250 insertions(+), 79 deletions(-)
discards 6ba68dd8cc60eaec5d71b319d7f4f9677931ddbc (commit)
via 743341f8a70b126b82ef3293bd9f6c2bc47ec29a (commit)
via d14d34b534e6914dea107878212436dd995c6560 (commit)
via 76683cdf4d90868944a51e6b433dce3ff8f5495e (commit)
via 94442974428c037df716204296a3579dbc645bbc (commit)
via 23c3347fe9779c3826179285de86e8540d46d749 (commit)
via 41694642af159d156751afc7dc964d46e0fd8a81 (commit)
via 73a127e5492bc2711530b2f5a7c30a5021232d40 (commit)
via d6f163ed99c6ae234c551bb3ecd23c7af8610c8f (commit)
via 1c859533008748afa273aa7c379ef5aec88d4ce6 (commit)
via 155b1aa3772eb2d9421c5bfc678320401eb3b058 (commit)
via 1d6bcbbdbc374aa25b70c85717f987d2675c1205 (commit)
via 04784067f7f39e15161cc165365aa2d0552dc219 (commit)
via dec876f5ea5d88ea9674edc01b86ed2a4fa95078 (commit)
via db55813cc9f24c699db1d3386a56bc0a36debab0 (commit)
via c112d8551b75514b9ccbde44ee3e37904051dbb1 (commit)
via f4d556b93687fc915897c9cec97afebd7b86572e (commit)
via f34c10a5b22e3f9941f2cfe60f6f9d7a81319b6f (commit)
via f3b78f5c0690a93d4cc9370519f970ded430c453 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (6ba68dd8cc60eaec5d71b319d7f4f9677931ddbc)
\
N -- N -- N (743341f8a70b126b82ef3293bd9f6c2bc47ec29a)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 743341f8a70b126b82ef3293bd9f6c2bc47ec29a
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Nov 20 15:36:04 2015 -0500
7711: Store cloud node size id and price in properties of Arvados node record.
diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
index 3c70877..1016fda 100644
--- a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
@@ -88,6 +88,7 @@ class ComputeNodeStateChangeBase(config.actor_class):
'slot_number': None,
'first_ping_at': None,
'last_ping_at': None,
+ 'properties': {},
'info': {'ec2_instance_id': None,
'last_action': explanation}},
).execute()
@@ -134,6 +135,30 @@ class ComputeNodeSetupActor(ComputeNodeStateChangeBase):
self.cloud_node = self._cloud.create_node(self.cloud_size,
self.arvados_node)
self._logger.info("Cloud node %s created.", self.cloud_node.id)
+ self._later.update_arvados_node_properties()
+
+ @ComputeNodeStateChangeBase._retry(config.ARVADOS_ERRORS)
+ def update_arvados_node_properties(self):
+ """Tell Arvados some details about the cloud node.
+
+ Currently we only include size/price from our request, which
+ we already knew before create_cloud_node(), but doing it here
+ gives us an opportunity to provide more detail from
+ self.cloud_node, too.
+ """
+ self.arvados_node['properties']['cloud_node'] = {
+ # Note this 'size' is the node size we asked the cloud
+ # driver to create -- not necessarily equal to the size
+ # reported by the cloud driver for the node that was
+ # created.
+ 'size': self.cloud_size.id,
+ 'price': self.cloud_size.price,
+ }
+ self.arvados_node = self._arvados.nodes().update(
+ uuid=self.arvados_node['uuid'],
+ body={'properties': self.arvados_node['properties']},
+ ).execute()
+ self._logger.info("%s updated properties.", self.arvados_node['uuid'])
self._later.post_create()
@ComputeNodeStateChangeBase._retry()
diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py
index 0bdb2cb..ecf83c6 100644
--- a/services/nodemanager/tests/test_computenode_dispatch.py
+++ b/services/nodemanager/tests/test_computenode_dispatch.py
@@ -27,24 +27,38 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
def make_actor(self, arv_node=None):
if not hasattr(self, 'timer'):
- self.make_mocks(arvados_effect=[arv_node])
+ self.make_mocks(arvados_effect=[arv_node] if arv_node else None)
self.setup_actor = dispatch.ComputeNodeSetupActor.start(
self.timer, self.api_client, self.cloud_client,
testutil.MockSize(1), arv_node).proxy()
+ def assert_node_properties_updated(self, uuid=None,
+ size=testutil.MockSize(1)):
+ self.api_client.nodes().update.assert_any_call(
+ uuid=(uuid or self.arvados_effect[-1]['uuid']),
+ body={
+ 'properties': {
+ 'cloud_node': {
+ 'size': size.id,
+ 'price': size.price}}})
+
def test_creation_without_arvados_node(self):
self.make_actor()
self.assertEqual(self.arvados_effect[-1],
self.setup_actor.arvados_node.get(self.TIMEOUT))
- self.assertTrue(self.api_client.nodes().create().execute.called)
+ self.assertEqual(1, self.api_client.nodes().create().execute.call_count)
+ self.assertEqual(1, self.api_client.nodes().update().execute.call_count)
+ self.assert_node_properties_updated()
self.assertEqual(self.cloud_client.create_node(),
self.setup_actor.cloud_node.get(self.TIMEOUT))
def test_creation_with_arvados_node(self):
+ self.make_mocks(arvados_effect=[testutil.arvados_node_mock()]*2)
self.make_actor(testutil.arvados_node_mock())
self.assertEqual(self.arvados_effect[-1],
self.setup_actor.arvados_node.get(self.TIMEOUT))
- self.assertTrue(self.api_client.nodes().update().execute.called)
+ self.assert_node_properties_updated()
+ self.assertEqual(2, self.api_client.nodes().update().execute.call_count)
self.assertEqual(self.cloud_client.create_node(),
self.setup_actor.cloud_node.get(self.TIMEOUT))
@@ -98,8 +112,9 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
self.make_actor()
subscriber = mock.Mock(name='subscriber_mock')
self.setup_actor.subscribe(subscriber)
- self.api_client.nodes().create().execute.side_effect = [
- testutil.arvados_node_mock()]
+ retry_resp = [testutil.arvados_node_mock()]
+ self.api_client.nodes().create().execute.side_effect = retry_resp
+ self.api_client.nodes().update().execute.side_effect = retry_resp
self.wait_for_assignment(self.setup_actor, 'cloud_node')
self.assertEqual(self.setup_actor.actor_ref.actor_urn,
subscriber.call_args[0][0].actor_ref.actor_urn)
diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py
index aeb9768..e543c28 100644
--- a/services/nodemanager/tests/testutil.py
+++ b/services/nodemanager/tests/testutil.py
@@ -30,6 +30,7 @@ def arvados_node_mock(node_num=99, job_uuid=None, age=-1, **kwargs):
'ip_address': ip_address_mock(node_num),
'job_uuid': job_uuid,
'crunch_worker_state': crunch_worker_state,
+ 'properties': {},
'info': {'ping_secret': 'defaulttestsecret'}}
node.update(kwargs)
return node
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list