[ARVADOS] updated: 1.1.4-402-g17f521d
Git user
git at public.curoverse.com
Mon Jun 11 16:52:29 EDT 2018
Summary of changes:
apps/workbench/test/integration_helper.rb | 2 +
build/run-tests.sh | 17 ++-
sdk/cwl/arvados_cwl/__init__.py | 7 --
sdk/cwl/arvados_cwl/http.py | 16 ++-
sdk/cwl/arvados_cwl/pathmapper.py | 21 +---
sdk/cwl/arvados_cwl/runner.py | 9 +-
sdk/cwl/setup.py | 2 +-
sdk/cwl/tests/test_pathmapper.py | 19 +---
sdk/go/keepclient/keepclient.go | 1 +
sdk/go/keepclient/keepclient_test.go | 23 +++-
sdk/go/keepclient/support.go | 3 +
sdk/python/arvados/collection.py | 90 ++++++++++++---
sdk/python/arvados/commands/run.py | 11 +-
sdk/python/tests/test_collections.py | 38 ++++++-
.../arvados/v1/containers_controller.rb | 6 +-
services/api/app/models/container.rb | 8 +-
services/api/app/models/container_request.rb | 2 -
services/api/lib/update_priority.rb | 38 +++++++
services/keep-balance/balance.go | 121 +++++++++++++++++----
services/keep-balance/balance_test.go | 93 +++++++++++++++-
services/keepproxy/keepproxy.go | 9 ++
services/keepproxy/keepproxy_test.go | 46 ++++++--
.../arvnodeman/computenode/dispatch/__init__.py | 5 +
.../arvnodeman/computenode/driver/azure.py | 10 +-
services/nodemanager/arvnodeman/config.py | 4 +-
services/nodemanager/arvnodeman/jobqueue.py | 22 +++-
services/nodemanager/arvnodeman/launcher.py | 2 +-
.../nodemanager/arvnodeman/test/fake_driver.py | 4 +-
services/nodemanager/tests/fake_azure.cfg.template | 8 +-
services/nodemanager/tests/fake_ec2.cfg.template | 8 +-
services/nodemanager/tests/fake_gce.cfg.template | 8 +-
services/nodemanager/tests/integration_test.py | 38 ++-----
.../nodemanager/tests/test_computenode_dispatch.py | 6 +
.../tests/test_computenode_driver_azure.py | 23 +++-
.../tests/test_computenode_driver_ec2.py | 27 ++++-
.../tests/test_computenode_driver_gce.py | 11 ++
services/nodemanager/tests/test_config.py | 21 +++-
services/nodemanager/tests/testutil.py | 4 +-
38 files changed, 594 insertions(+), 189 deletions(-)
create mode 100644 services/api/lib/update_priority.rb
via 17f521d7ffb4f3a58ca98a27395eb60d9fa34519 (commit)
via 5f42faf0818c229196b26535a0669eeb2bf27386 (commit)
via a4991cffc83afea213125a24f01399d6f5707948 (commit)
via 8e31910034627dedd7259dd1e45a60768108c1e1 (commit)
via d2efca1759c9e104f67d1caf85c8e62ff06807bd (commit)
via 85c13201abf4a9f19dd112992a04fa65da63369c (commit)
via 333a30289f47c22956bcb6a045d1cd4c797e22c3 (commit)
via 31fd95388f035bba3e70d3e8861a14511f6281e6 (commit)
via 7d7eb76d319f1ab6faeb8634f13142e7147a45ea (commit)
via 8f4b1051209f78bff358069a2a3b2f646f7a38f8 (commit)
via e3483661acc3d26d4532a536bd40ed761c21d7d1 (commit)
via 1dc6c511620a852f81d5179fd2f9fbecee472e71 (commit)
via 57fcaf45fe40806dca5f001aabed9b413243183e (commit)
via afbc818e5bbc667b69e9ae9b8f080cb50ca17c53 (commit)
via ad5f111b67e2bf1c43ce9bc37e6b11b8c753f62d (commit)
via 26d20f13895fdf5c9224a7aca2b33625c511e73e (commit)
via da40bd0960806df8e2799e4fb716d41ad08b169f (commit)
via 1714431955ac7b2a1d2fd3c2cda9b806bad624cb (commit)
via da2cfffb3a3ec92c3b15841255dc704a99748fea (commit)
via df8a51cd6b8a95475dfaef08c8a56b1e1643dbfc (commit)
via 1b183730bd6ec56d51b17020e63fb2cffa2ad764 (commit)
via c9143544609d90da33eb3c2d566fc5d6a25188b2 (commit)
via d5dd5dc2838fe420c8f835975bad7052a0df55e5 (commit)
via 0809440ad583a2556c0f97fd000be9e9b7f94eb5 (commit)
via 7654764e5192fc8d34e612a6ee6608fad33a91c2 (commit)
via 0eedf70afa34167275d2135837e866b13fac4178 (commit)
via cae5f7def9dae9caf9056bc3bb77ca801cfa2229 (commit)
via e80efb5caae2acbe50ea8a1bd1151b0d2a67c1c9 (commit)
via 7b7950766b7a77cd6c2d6ecee7603dd7394c4144 (commit)
via 24af6ae1b23a4fd82c816d077c0178f0d75db2d7 (commit)
via 4869ae0f12b5a4a293c66f7304483fee1addae52 (commit)
via 7e6ac5a4967614cbe59ed5c0ec41c8be4d4cff4d (commit)
via 4f0c3d501d19bed5915d5d188598d3a7f1dec7f8 (commit)
via dd97299b6b3ce82b6a57e2838fbe3f81bb51f140 (commit)
via ed7698518d7eabea0f4e86acfb0660765fe31071 (commit)
via 783523246354d49b5cba8c36bb2913831e5f8e52 (commit)
via 32568bbde8ad549452ccb57f10bb54672fda6fa6 (commit)
via 6c5fc89c06338ea829f2e4f2a7e0529548890efd (commit)
via 3108e6a627f4413819af8d2eb006016f1695b9a8 (commit)
via e1ee43ee90b7f6e6189930cb031448014d0cf638 (commit)
via d9462226315eb4a479b6bb998aa51531062635b0 (commit)
via 6814a151d680775d9804d56582a94af90cee2ab8 (commit)
via 49bb6cb300e9f6a3862c4e17112e34da1226a3c7 (commit)
via f57c43c811ee8af08e7662ad567a6d8e8c18b781 (commit)
via 3c2266151711f532491a2369de37086bd7234dce (commit)
from 1965e1793598c76ceef4decd4a5a6c7755c385e3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 17f521d7ffb4f3a58ca98a27395eb60d9fa34519
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Jun 11 17:51:38 2018 -0300
7478: Adds test to check that state is 'down' with 'invalid' size.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py
index 5775aa6..2849265 100644
--- a/services/nodemanager/tests/test_computenode_dispatch.py
+++ b/services/nodemanager/tests/test_computenode_dispatch.py
@@ -356,6 +356,12 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
def node_state(self, *states):
return self.node_actor.in_state(*states).get(self.TIMEOUT)
+ def test_in_state_when_invalid_cloud_node_size(self):
+ self.make_mocks(1)
+ self.cloud_mock.size.id = 'invalid'
+ self.make_actor()
+ self.assertTrue(self.node_state('down'))
+
def test_in_state_when_unpaired(self):
self.make_actor()
self.assertTrue(self.node_state('unpaired'))
commit 5f42faf0818c229196b26535a0669eeb2bf27386
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Jun 11 16:47:50 2018 -0300
7478: Adds tests on driver's create_node calls.
* Check that arvados_node_size tag is passed
* On EC2 driver, check that ex_spot_market=True is used when requesting
spot instances.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/computenode/driver/azure.py b/services/nodemanager/arvnodeman/computenode/driver/azure.py
index 7c299db..ae55432 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/azure.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/azure.py
@@ -46,6 +46,8 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
def arvados_create_kwargs(self, size, arvados_node):
tags = {
+ # Set up tag indicating the Arvados assigned Cloud Size id.
+ 'arvados_node_size': size.id,
'booted_at': time.strftime(ARVADOS_TIMEFMT, time.gmtime()),
'arv-ping-url': self._make_ping_url(arvados_node)
}
@@ -78,12 +80,6 @@ echo %s > /var/tmp/arv-node-data/meta-data/instance-type
def _init_image(self, urn):
return "image", self.get_image(urn)
- def create_node(self, size, arvados_node):
- # Set up tag indicating the Arvados assigned Cloud Size id.
- self.create_kwargs.setdefault('ex_tags', {})
- self.create_kwargs['ex_tags'].update({'arvados_node_size': size.id})
- return super(ComputeNodeDriver, self).create_node(size, arvados_node)
-
def list_nodes(self):
# Azure only supports filtering node lists by resource group.
# Do our own filtering based on tag.
diff --git a/services/nodemanager/tests/test_computenode_driver_azure.py b/services/nodemanager/tests/test_computenode_driver_azure.py
index 87cbc2e..ea7a033 100644
--- a/services/nodemanager/tests/test_computenode_driver_azure.py
+++ b/services/nodemanager/tests/test_computenode_driver_azure.py
@@ -44,14 +44,25 @@ class AzureComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase
self.assertIn('ping_secret=ssshh',
create_method.call_args[1].get('ex_tags', {}).get('arv-ping-url', ""))
+ def test_create_includes_arvados_node_size(self):
+ arv_node = testutil.arvados_node_mock()
+ arv_node["hostname"] = None
+ size = testutil.MockSize(1)
+ driver = self.new_driver()
+ driver.create_node(size, arv_node)
+ create_method = self.driver_mock().create_node
+ self.assertTrue(create_method.called)
+ self.assertIn(
+ ('arvados_node_size', size.id),
+ create_method.call_args[1].get('ex_tags', {'tags': 'missing'}).items()
+ )
+
def test_name_from_new_arvados_node(self):
arv_node = testutil.arvados_node_mock(hostname=None)
driver = self.new_driver()
self.assertEqual('compute-000000000000063-zzzzz',
driver.arvados_create_kwargs(testutil.MockSize(1), arv_node)['name'])
-
-
def check_node_tagged(self, cloud_node, expected_tags):
tag_mock = self.driver_mock().ex_create_tags
self.assertTrue(tag_mock.called)
diff --git a/services/nodemanager/tests/test_computenode_driver_ec2.py b/services/nodemanager/tests/test_computenode_driver_ec2.py
index ed0e0bd..9442a8c 100644
--- a/services/nodemanager/tests/test_computenode_driver_ec2.py
+++ b/services/nodemanager/tests/test_computenode_driver_ec2.py
@@ -58,9 +58,30 @@ class EC2ComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
self.assertTrue(create_method.called)
self.assertIn(
('test', 'testvalue'),
- create_method.call_args[1].get(
- 'ex_metadata',
- {'arg': 'missing'}).items()
+ create_method.call_args[1].get('ex_metadata', {'arg': 'missing'}).items()
+ )
+
+ def test_create_includes_arvados_node_size(self):
+ arv_node = testutil.arvados_node_mock()
+ size = testutil.MockSize(1)
+ driver = self.new_driver()
+ driver.create_node(size, arv_node)
+ create_method = self.driver_mock().create_node
+ self.assertTrue(create_method.called)
+ self.assertIn(
+ ('arvados_node_size', size.id),
+ create_method.call_args[1].get('ex_metadata', {'arg': 'missing'}).items()
+ )
+
+ def test_create_preemptable_instance(self):
+ arv_node = testutil.arvados_node_mock()
+ driver = self.new_driver()
+ driver.create_node(testutil.MockSize(1, preemptable=True), arv_node)
+ create_method = self.driver_mock().create_node
+ self.assertTrue(create_method.called)
+ self.assertEqual(
+ True,
+ create_method.call_args[1].get('ex_spot_market', 'arg missing')
)
def test_hostname_from_arvados_node(self):
diff --git a/services/nodemanager/tests/test_computenode_driver_gce.py b/services/nodemanager/tests/test_computenode_driver_gce.py
index f0942e9..1446cd2 100644
--- a/services/nodemanager/tests/test_computenode_driver_gce.py
+++ b/services/nodemanager/tests/test_computenode_driver_gce.py
@@ -51,6 +51,17 @@ class GCEComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
metadata = self.driver_mock().create_node.call_args[1]['ex_metadata']
self.assertIn('ping_secret=ssshh', metadata.get('arv-ping-url'))
+ def test_create_includes_arvados_node_size(self):
+ arv_node = testutil.arvados_node_mock()
+ size = testutil.MockSize(1)
+ driver = self.new_driver()
+ driver.create_node(size, arv_node)
+ create_method = self.driver_mock().create_node
+ self.assertIn(
+ ('arvados_node_size', size.id),
+ create_method.call_args[1].get('ex_metadata', {'metadata':'missing'}).items()
+ )
+
def test_create_raises_but_actually_succeeded(self):
arv_node = testutil.arvados_node_mock(1, hostname=None)
driver = self.new_driver()
diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py
index 857e09f..2ec13c0 100644
--- a/services/nodemanager/tests/testutil.py
+++ b/services/nodemanager/tests/testutil.py
@@ -78,7 +78,7 @@ class MockShutdownTimer(object):
class MockSize(object):
- def __init__(self, factor):
+ def __init__(self, factor, preemptable=False):
self.id = 'z{}.test'.format(factor)
self.name = 'test size '+self.id
self.ram = 128 * factor
@@ -88,7 +88,7 @@ class MockSize(object):
self.price = float(factor)
self.extra = {}
self.real = self
- self.preemptable = False
+ self.preemptable = preemptable
def __eq__(self, other):
return self.id == other.id
commit a4991cffc83afea213125a24f01399d6f5707948
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Jun 11 13:59:31 2018 -0300
7478: Config load fix & test updates about instance type decoupling
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index 4171b74..f9724a8 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -72,6 +72,7 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
def get_section(self, section, transformers={}, default_transformer=None):
transformer_map = {
+ str: self.get,
int: self.getint,
bool: self.getboolean,
float: self.getfloat,
@@ -148,6 +149,7 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
all_sizes = self.new_cloud_client().list_sizes()
size_kwargs = {}
section_types = {
+ 'instance_type': str,
'price': float,
'preemptable': bool,
}
diff --git a/services/nodemanager/tests/test_config.py b/services/nodemanager/tests/test_config.py
index 15716d5..9a48c7c 100644
--- a/services/nodemanager/tests/test_config.py
+++ b/services/nodemanager/tests/test_config.py
@@ -29,6 +29,12 @@ creds = dummy_creds
cores = 1
price = 0.8
+[Size 1.preemptable]
+instance_type = 1
+preemptable = true
+cores = 1
+price = 0.8
+
[Logging]
file = /dev/null
level = DEBUG
@@ -54,11 +60,24 @@ testlogger = INFO
def test_list_sizes(self):
config = self.load_config()
sizes = config.node_sizes()
- self.assertEqual(1, len(sizes))
+ self.assertEqual(2, len(sizes))
size, kwargs = sizes[0]
self.assertEqual('Small', size.name)
self.assertEqual(1, kwargs['cores'])
self.assertEqual(0.8, kwargs['price'])
+ # preemptable is False by default
+ self.assertEqual(False, kwargs['preemptable'])
+ # instance_type == arvados node size id by default
+ self.assertEqual(kwargs['id'], kwargs['instance_type'])
+ # Now retrieve the preemptable version
+ size, kwargs = sizes[1]
+ self.assertEqual('Small', size.name)
+ self.assertEqual('1.preemptable', kwargs['id'])
+ self.assertEqual(1, kwargs['cores'])
+ self.assertEqual(0.8, kwargs['price'])
+ self.assertEqual(True, kwargs['preemptable'])
+ self.assertEqual('1', kwargs['instance_type'])
+
def test_default_node_mem_scaling(self):
config = self.load_config()
commit 8e31910034627dedd7259dd1e45a60768108c1e1
Merge: d2efca1 85c1320
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Jun 11 12:34:22 2018 -0300
Merge branch 'master' into 7478-anm-spot-instances
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
commit d2efca1759c9e104f67d1caf85c8e62ff06807bd
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Jun 11 12:10:44 2018 -0300
7478: Assign invalid cloud sizes to nodes with stale arvados_node_size tags
When retrieving the node's list from the cloud provider, if the tag
refers to a non existant arvados cloud size, assign an invalid size
and return 'down' state when asked so that it can be properly shut down.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
index 9106ea6..dec8ef1 100644
--- a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
@@ -370,6 +370,11 @@ class ComputeNodeMonitorActor(config.actor_class):
def get_state(self):
"""Get node state, one of ['unpaired', 'busy', 'idle', 'down']."""
+ # If this node's size is invalid (because it has a stale arvados_node_size
+ # tag), return 'down' so that it's properly shut down.
+ if self.cloud_node.size.id == 'invalid':
+ return 'down'
+
# If this node is not associated with an Arvados node, return
# 'unpaired' if we're in the boot grace period, and 'down' if not,
# so it isn't counted towards usable nodes.
diff --git a/services/nodemanager/arvnodeman/jobqueue.py b/services/nodemanager/arvnodeman/jobqueue.py
index f3fdc8e..db578e0 100644
--- a/services/nodemanager/arvnodeman/jobqueue.py
+++ b/services/nodemanager/arvnodeman/jobqueue.py
@@ -24,6 +24,26 @@ class ServerCalculator(object):
that would best satisfy the jobs, choosing the cheapest size that
satisfies each job, and ignoring jobs that can't be satisfied.
"""
+ class InvalidCloudSize(object):
+ """
+ Dummy CloudSizeWrapper-like class, to be used when a cloud node doesn't
+ have a recognizable arvados_node_size tag.
+ """
+ def __init__(self):
+ self.id = 'invalid'
+ self.name = 'invalid'
+ self.ram = 0
+ self.disk = 0
+ self.scratch = 0
+ self.cores = 0
+ self.bandwidth = 0
+ self.price = 9999999
+ self.preemptable = False
+ self.extra = {}
+
+ def meets_constraints(self, **kwargs):
+ return False
+
class CloudSizeWrapper(object):
def __init__(self, real_size, node_mem_scaling, **kwargs):
@@ -119,7 +139,7 @@ class ServerCalculator(object):
for s in self.cloud_sizes:
if s.id == sizeid:
return s
- return None
+ return InvalidCloudSize()
class JobQueueMonitorActor(clientactor.RemotePollLoopActor):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list