[ARVADOS] updated: 1.1.4-402-g17f521d

Git user git at public.curoverse.com
Mon Jun 11 16:52:29 EDT 2018


Summary of changes:
 apps/workbench/test/integration_helper.rb          |   2 +
 build/run-tests.sh                                 |  17 ++-
 sdk/cwl/arvados_cwl/__init__.py                    |   7 --
 sdk/cwl/arvados_cwl/http.py                        |  16 ++-
 sdk/cwl/arvados_cwl/pathmapper.py                  |  21 +---
 sdk/cwl/arvados_cwl/runner.py                      |   9 +-
 sdk/cwl/setup.py                                   |   2 +-
 sdk/cwl/tests/test_pathmapper.py                   |  19 +---
 sdk/go/keepclient/keepclient.go                    |   1 +
 sdk/go/keepclient/keepclient_test.go               |  23 +++-
 sdk/go/keepclient/support.go                       |   3 +
 sdk/python/arvados/collection.py                   |  90 ++++++++++++---
 sdk/python/arvados/commands/run.py                 |  11 +-
 sdk/python/tests/test_collections.py               |  38 ++++++-
 .../arvados/v1/containers_controller.rb            |   6 +-
 services/api/app/models/container.rb               |   8 +-
 services/api/app/models/container_request.rb       |   2 -
 services/api/lib/update_priority.rb                |  38 +++++++
 services/keep-balance/balance.go                   | 121 +++++++++++++++++----
 services/keep-balance/balance_test.go              |  93 +++++++++++++++-
 services/keepproxy/keepproxy.go                    |   9 ++
 services/keepproxy/keepproxy_test.go               |  46 ++++++--
 .../arvnodeman/computenode/dispatch/__init__.py    |   5 +
 .../arvnodeman/computenode/driver/azure.py         |  10 +-
 services/nodemanager/arvnodeman/config.py          |   4 +-
 services/nodemanager/arvnodeman/jobqueue.py        |  22 +++-
 services/nodemanager/arvnodeman/launcher.py        |   2 +-
 .../nodemanager/arvnodeman/test/fake_driver.py     |   4 +-
 services/nodemanager/tests/fake_azure.cfg.template |   8 +-
 services/nodemanager/tests/fake_ec2.cfg.template   |   8 +-
 services/nodemanager/tests/fake_gce.cfg.template   |   8 +-
 services/nodemanager/tests/integration_test.py     |  38 ++-----
 .../nodemanager/tests/test_computenode_dispatch.py |   6 +
 .../tests/test_computenode_driver_azure.py         |  23 +++-
 .../tests/test_computenode_driver_ec2.py           |  27 ++++-
 .../tests/test_computenode_driver_gce.py           |  11 ++
 services/nodemanager/tests/test_config.py          |  21 +++-
 services/nodemanager/tests/testutil.py             |   4 +-
 38 files changed, 594 insertions(+), 189 deletions(-)
 create mode 100644 services/api/lib/update_priority.rb

       via  17f521d7ffb4f3a58ca98a27395eb60d9fa34519 (commit)
       via  5f42faf0818c229196b26535a0669eeb2bf27386 (commit)
       via  a4991cffc83afea213125a24f01399d6f5707948 (commit)
       via  8e31910034627dedd7259dd1e45a60768108c1e1 (commit)
       via  d2efca1759c9e104f67d1caf85c8e62ff06807bd (commit)
       via  85c13201abf4a9f19dd112992a04fa65da63369c (commit)
       via  333a30289f47c22956bcb6a045d1cd4c797e22c3 (commit)
       via  31fd95388f035bba3e70d3e8861a14511f6281e6 (commit)
       via  7d7eb76d319f1ab6faeb8634f13142e7147a45ea (commit)
       via  8f4b1051209f78bff358069a2a3b2f646f7a38f8 (commit)
       via  e3483661acc3d26d4532a536bd40ed761c21d7d1 (commit)
       via  1dc6c511620a852f81d5179fd2f9fbecee472e71 (commit)
       via  57fcaf45fe40806dca5f001aabed9b413243183e (commit)
       via  afbc818e5bbc667b69e9ae9b8f080cb50ca17c53 (commit)
       via  ad5f111b67e2bf1c43ce9bc37e6b11b8c753f62d (commit)
       via  26d20f13895fdf5c9224a7aca2b33625c511e73e (commit)
       via  da40bd0960806df8e2799e4fb716d41ad08b169f (commit)
       via  1714431955ac7b2a1d2fd3c2cda9b806bad624cb (commit)
       via  da2cfffb3a3ec92c3b15841255dc704a99748fea (commit)
       via  df8a51cd6b8a95475dfaef08c8a56b1e1643dbfc (commit)
       via  1b183730bd6ec56d51b17020e63fb2cffa2ad764 (commit)
       via  c9143544609d90da33eb3c2d566fc5d6a25188b2 (commit)
       via  d5dd5dc2838fe420c8f835975bad7052a0df55e5 (commit)
       via  0809440ad583a2556c0f97fd000be9e9b7f94eb5 (commit)
       via  7654764e5192fc8d34e612a6ee6608fad33a91c2 (commit)
       via  0eedf70afa34167275d2135837e866b13fac4178 (commit)
       via  cae5f7def9dae9caf9056bc3bb77ca801cfa2229 (commit)
       via  e80efb5caae2acbe50ea8a1bd1151b0d2a67c1c9 (commit)
       via  7b7950766b7a77cd6c2d6ecee7603dd7394c4144 (commit)
       via  24af6ae1b23a4fd82c816d077c0178f0d75db2d7 (commit)
       via  4869ae0f12b5a4a293c66f7304483fee1addae52 (commit)
       via  7e6ac5a4967614cbe59ed5c0ec41c8be4d4cff4d (commit)
       via  4f0c3d501d19bed5915d5d188598d3a7f1dec7f8 (commit)
       via  dd97299b6b3ce82b6a57e2838fbe3f81bb51f140 (commit)
       via  ed7698518d7eabea0f4e86acfb0660765fe31071 (commit)
       via  783523246354d49b5cba8c36bb2913831e5f8e52 (commit)
       via  32568bbde8ad549452ccb57f10bb54672fda6fa6 (commit)
       via  6c5fc89c06338ea829f2e4f2a7e0529548890efd (commit)
       via  3108e6a627f4413819af8d2eb006016f1695b9a8 (commit)
       via  e1ee43ee90b7f6e6189930cb031448014d0cf638 (commit)
       via  d9462226315eb4a479b6bb998aa51531062635b0 (commit)
       via  6814a151d680775d9804d56582a94af90cee2ab8 (commit)
       via  49bb6cb300e9f6a3862c4e17112e34da1226a3c7 (commit)
       via  f57c43c811ee8af08e7662ad567a6d8e8c18b781 (commit)
       via  3c2266151711f532491a2369de37086bd7234dce (commit)
      from  1965e1793598c76ceef4decd4a5a6c7755c385e3 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 17f521d7ffb4f3a58ca98a27395eb60d9fa34519
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Mon Jun 11 17:51:38 2018 -0300

    7478: Adds test to check that state is 'down' with 'invalid' size.
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>

diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py
index 5775aa6..2849265 100644
--- a/services/nodemanager/tests/test_computenode_dispatch.py
+++ b/services/nodemanager/tests/test_computenode_dispatch.py
@@ -356,6 +356,12 @@ class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
     def node_state(self, *states):
         return self.node_actor.in_state(*states).get(self.TIMEOUT)
 
+    def test_in_state_when_invalid_cloud_node_size(self):
+        self.make_mocks(1)
+        self.cloud_mock.size.id = 'invalid'
+        self.make_actor()
+        self.assertTrue(self.node_state('down'))
+
     def test_in_state_when_unpaired(self):
         self.make_actor()
         self.assertTrue(self.node_state('unpaired'))

commit 5f42faf0818c229196b26535a0669eeb2bf27386
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Mon Jun 11 16:47:50 2018 -0300

    7478: Adds tests on driver's create_node calls.
    
    * Check that arvados_node_size tag is passed
    * On EC2 driver, check that ex_spot_market=True is used when requesting
    spot instances.
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>

diff --git a/services/nodemanager/arvnodeman/computenode/driver/azure.py b/services/nodemanager/arvnodeman/computenode/driver/azure.py
index 7c299db..ae55432 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/azure.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/azure.py
@@ -46,6 +46,8 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
 
     def arvados_create_kwargs(self, size, arvados_node):
         tags = {
+            # Set up tag indicating the Arvados assigned Cloud Size id.
+            'arvados_node_size': size.id,
             'booted_at': time.strftime(ARVADOS_TIMEFMT, time.gmtime()),
             'arv-ping-url': self._make_ping_url(arvados_node)
         }
@@ -78,12 +80,6 @@ echo %s > /var/tmp/arv-node-data/meta-data/instance-type
     def _init_image(self, urn):
         return "image", self.get_image(urn)
 
-    def create_node(self, size, arvados_node):
-        # Set up tag indicating the Arvados assigned Cloud Size id.
-        self.create_kwargs.setdefault('ex_tags', {})
-        self.create_kwargs['ex_tags'].update({'arvados_node_size': size.id})
-        return super(ComputeNodeDriver, self).create_node(size, arvados_node)
-
     def list_nodes(self):
         # Azure only supports filtering node lists by resource group.
         # Do our own filtering based on tag.
diff --git a/services/nodemanager/tests/test_computenode_driver_azure.py b/services/nodemanager/tests/test_computenode_driver_azure.py
index 87cbc2e..ea7a033 100644
--- a/services/nodemanager/tests/test_computenode_driver_azure.py
+++ b/services/nodemanager/tests/test_computenode_driver_azure.py
@@ -44,14 +44,25 @@ class AzureComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase
         self.assertIn('ping_secret=ssshh',
                       create_method.call_args[1].get('ex_tags', {}).get('arv-ping-url', ""))
 
+    def test_create_includes_arvados_node_size(self):
+        arv_node = testutil.arvados_node_mock()
+        arv_node["hostname"] = None
+        size = testutil.MockSize(1)
+        driver = self.new_driver()
+        driver.create_node(size, arv_node)
+        create_method = self.driver_mock().create_node
+        self.assertTrue(create_method.called)
+        self.assertIn(
+            ('arvados_node_size', size.id),
+            create_method.call_args[1].get('ex_tags', {'tags': 'missing'}).items()
+        )
+
     def test_name_from_new_arvados_node(self):
         arv_node = testutil.arvados_node_mock(hostname=None)
         driver = self.new_driver()
         self.assertEqual('compute-000000000000063-zzzzz',
                          driver.arvados_create_kwargs(testutil.MockSize(1), arv_node)['name'])
 
-
-
     def check_node_tagged(self, cloud_node, expected_tags):
         tag_mock = self.driver_mock().ex_create_tags
         self.assertTrue(tag_mock.called)
diff --git a/services/nodemanager/tests/test_computenode_driver_ec2.py b/services/nodemanager/tests/test_computenode_driver_ec2.py
index ed0e0bd..9442a8c 100644
--- a/services/nodemanager/tests/test_computenode_driver_ec2.py
+++ b/services/nodemanager/tests/test_computenode_driver_ec2.py
@@ -58,9 +58,30 @@ class EC2ComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
         self.assertTrue(create_method.called)
         self.assertIn(
             ('test', 'testvalue'),
-            create_method.call_args[1].get(
-                'ex_metadata',
-                {'arg': 'missing'}).items()
+            create_method.call_args[1].get('ex_metadata', {'arg': 'missing'}).items()
+        )
+
+    def test_create_includes_arvados_node_size(self):
+        arv_node = testutil.arvados_node_mock()
+        size = testutil.MockSize(1)
+        driver = self.new_driver()
+        driver.create_node(size, arv_node)
+        create_method = self.driver_mock().create_node
+        self.assertTrue(create_method.called)
+        self.assertIn(
+            ('arvados_node_size', size.id),
+            create_method.call_args[1].get('ex_metadata', {'arg': 'missing'}).items()
+        )
+
+    def test_create_preemptable_instance(self):
+        arv_node = testutil.arvados_node_mock()
+        driver = self.new_driver()
+        driver.create_node(testutil.MockSize(1, preemptable=True), arv_node)
+        create_method = self.driver_mock().create_node
+        self.assertTrue(create_method.called)
+        self.assertEqual(
+            True,
+            create_method.call_args[1].get('ex_spot_market', 'arg missing')
         )
 
     def test_hostname_from_arvados_node(self):
diff --git a/services/nodemanager/tests/test_computenode_driver_gce.py b/services/nodemanager/tests/test_computenode_driver_gce.py
index f0942e9..1446cd2 100644
--- a/services/nodemanager/tests/test_computenode_driver_gce.py
+++ b/services/nodemanager/tests/test_computenode_driver_gce.py
@@ -51,6 +51,17 @@ class GCEComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
         metadata = self.driver_mock().create_node.call_args[1]['ex_metadata']
         self.assertIn('ping_secret=ssshh', metadata.get('arv-ping-url'))
 
+    def test_create_includes_arvados_node_size(self):
+        arv_node = testutil.arvados_node_mock()
+        size = testutil.MockSize(1)
+        driver = self.new_driver()
+        driver.create_node(size, arv_node)
+        create_method = self.driver_mock().create_node
+        self.assertIn(
+            ('arvados_node_size', size.id),
+            create_method.call_args[1].get('ex_metadata', {'metadata':'missing'}).items()
+        )
+
     def test_create_raises_but_actually_succeeded(self):
         arv_node = testutil.arvados_node_mock(1, hostname=None)
         driver = self.new_driver()
diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py
index 857e09f..2ec13c0 100644
--- a/services/nodemanager/tests/testutil.py
+++ b/services/nodemanager/tests/testutil.py
@@ -78,7 +78,7 @@ class MockShutdownTimer(object):
 
 
 class MockSize(object):
-    def __init__(self, factor):
+    def __init__(self, factor, preemptable=False):
         self.id = 'z{}.test'.format(factor)
         self.name = 'test size '+self.id
         self.ram = 128 * factor
@@ -88,7 +88,7 @@ class MockSize(object):
         self.price = float(factor)
         self.extra = {}
         self.real = self
-        self.preemptable = False
+        self.preemptable = preemptable
 
     def __eq__(self, other):
         return self.id == other.id

commit a4991cffc83afea213125a24f01399d6f5707948
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Mon Jun 11 13:59:31 2018 -0300

    7478: Config load fix & test updates about instance type decoupling
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>

diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index 4171b74..f9724a8 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -72,6 +72,7 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
 
     def get_section(self, section, transformers={}, default_transformer=None):
         transformer_map = {
+            str: self.get,
             int: self.getint,
             bool: self.getboolean,
             float: self.getfloat,
@@ -148,6 +149,7 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
         all_sizes = self.new_cloud_client().list_sizes()
         size_kwargs = {}
         section_types = {
+            'instance_type': str,
             'price': float,
             'preemptable': bool,
         }
diff --git a/services/nodemanager/tests/test_config.py b/services/nodemanager/tests/test_config.py
index 15716d5..9a48c7c 100644
--- a/services/nodemanager/tests/test_config.py
+++ b/services/nodemanager/tests/test_config.py
@@ -29,6 +29,12 @@ creds = dummy_creds
 cores = 1
 price = 0.8
 
+[Size 1.preemptable]
+instance_type = 1
+preemptable = true
+cores = 1
+price = 0.8
+
 [Logging]
 file = /dev/null
 level = DEBUG
@@ -54,11 +60,24 @@ testlogger = INFO
     def test_list_sizes(self):
         config = self.load_config()
         sizes = config.node_sizes()
-        self.assertEqual(1, len(sizes))
+        self.assertEqual(2, len(sizes))
         size, kwargs = sizes[0]
         self.assertEqual('Small', size.name)
         self.assertEqual(1, kwargs['cores'])
         self.assertEqual(0.8, kwargs['price'])
+        # preemptable is False by default
+        self.assertEqual(False, kwargs['preemptable'])
+        # instance_type == arvados node size id by default
+        self.assertEqual(kwargs['id'], kwargs['instance_type'])
+        # Now retrieve the preemptable version
+        size, kwargs = sizes[1]
+        self.assertEqual('Small', size.name)
+        self.assertEqual('1.preemptable', kwargs['id'])
+        self.assertEqual(1, kwargs['cores'])
+        self.assertEqual(0.8, kwargs['price'])
+        self.assertEqual(True, kwargs['preemptable'])
+        self.assertEqual('1', kwargs['instance_type'])
+
 
     def test_default_node_mem_scaling(self):
         config = self.load_config()

commit 8e31910034627dedd7259dd1e45a60768108c1e1
Merge: d2efca1 85c1320
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Mon Jun 11 12:34:22 2018 -0300

    Merge branch 'master' into 7478-anm-spot-instances
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>


commit d2efca1759c9e104f67d1caf85c8e62ff06807bd
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date:   Mon Jun 11 12:10:44 2018 -0300

    7478: Assign invalid cloud sizes to nodes with stale arvados_node_size tags
    
    When retrieving the node's list from the cloud provider, if the tag
    refers to a non existant arvados cloud size, assign an invalid size
    and return 'down' state when asked so that it can be properly shut down.
    
    Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>

diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
index 9106ea6..dec8ef1 100644
--- a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
@@ -370,6 +370,11 @@ class ComputeNodeMonitorActor(config.actor_class):
     def get_state(self):
         """Get node state, one of ['unpaired', 'busy', 'idle', 'down']."""
 
+        # If this node's size is invalid (because it has a stale arvados_node_size
+        # tag), return 'down' so that it's properly shut down.
+        if self.cloud_node.size.id == 'invalid':
+            return 'down'
+
         # If this node is not associated with an Arvados node, return
         # 'unpaired' if we're in the boot grace period, and 'down' if not,
         # so it isn't counted towards usable nodes.
diff --git a/services/nodemanager/arvnodeman/jobqueue.py b/services/nodemanager/arvnodeman/jobqueue.py
index f3fdc8e..db578e0 100644
--- a/services/nodemanager/arvnodeman/jobqueue.py
+++ b/services/nodemanager/arvnodeman/jobqueue.py
@@ -24,6 +24,26 @@ class ServerCalculator(object):
     that would best satisfy the jobs, choosing the cheapest size that
     satisfies each job, and ignoring jobs that can't be satisfied.
     """
+    class InvalidCloudSize(object):
+        """
+        Dummy CloudSizeWrapper-like class, to be used when a cloud node doesn't
+        have a recognizable arvados_node_size tag.
+        """
+        def __init__(self):
+            self.id = 'invalid'
+            self.name = 'invalid'
+            self.ram = 0
+            self.disk = 0
+            self.scratch = 0
+            self.cores = 0
+            self.bandwidth = 0
+            self.price = 9999999
+            self.preemptable = False
+            self.extra = {}
+
+        def meets_constraints(self, **kwargs):
+            return False
+
 
     class CloudSizeWrapper(object):
         def __init__(self, real_size, node_mem_scaling, **kwargs):
@@ -119,7 +139,7 @@ class ServerCalculator(object):
         for s in self.cloud_sizes:
             if s.id == sizeid:
                 return s
-        return None
+        return InvalidCloudSize()
 
 
 class JobQueueMonitorActor(clientactor.RemotePollLoopActor):

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list