[ARVADOS] created: 1.1.4-447-ga8bfbac31
Git user
git at public.curoverse.com
Tue Jun 19 13:03:38 EDT 2018
at a8bfbac314335eb0bff3f4ff5e856d3c327de31d (commit)
commit a8bfbac314335eb0bff3f4ff5e856d3c327de31d
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Tue Jun 19 14:02:07 2018 -0300
7478: Replaces term 'preemptable' with 'preemptible'
Also added config & documentation on EC2 example config file.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index b5fd0262a..4329f4f13 100644
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -62,7 +62,7 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
case it.Scratch < needScratch:
case it.RAM < needRAM:
case it.VCPUs < needVCPUs:
- case it.Preemptable != ctr.SchedulingParameters.Preemptable:
+ case it.Preemptible != ctr.SchedulingParameters.Preemptible:
case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs):
// Equal price, but worse specs
default:
diff --git a/lib/dispatchcloud/node_size_test.go b/lib/dispatchcloud/node_size_test.go
index d6b7c6bf9..1484f07a2 100644
--- a/lib/dispatchcloud/node_size_test.go
+++ b/lib/dispatchcloud/node_size_test.go
@@ -92,12 +92,12 @@ func (*NodeSizeSuite) TestChoose(c *check.C) {
}
}
-func (*NodeSizeSuite) TestChoosePreemptable(c *check.C) {
+func (*NodeSizeSuite) TestChoosePreemptible(c *check.C) {
menu := []arvados.InstanceType{
- {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Preemptable: true, Name: "costly"},
+ {Price: 4.4, RAM: 4000000000, VCPUs: 8, Scratch: 2 * GiB, Preemptible: true, Name: "costly"},
{Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "almost best"},
- {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Preemptable: true, Name: "best"},
- {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Preemptable: true, Name: "small"},
+ {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Preemptible: true, Name: "best"},
+ {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 2 * GiB, Preemptible: true, Name: "small"},
}
best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu}, &arvados.Container{
Mounts: map[string]arvados.Mount{
@@ -109,7 +109,7 @@ func (*NodeSizeSuite) TestChoosePreemptable(c *check.C) {
KeepCacheRAM: 123456789,
},
SchedulingParameters: arvados.SchedulingParameters{
- Preemptable: true,
+ Preemptible: true,
},
})
c.Check(err, check.IsNil)
@@ -117,5 +117,5 @@ func (*NodeSizeSuite) TestChoosePreemptable(c *check.C) {
c.Check(best.RAM >= 1234567890, check.Equals, true)
c.Check(best.VCPUs >= 2, check.Equals, true)
c.Check(best.Scratch >= 2*GiB, check.Equals, true)
- c.Check(best.Preemptable, check.Equals, true)
+ c.Check(best.Preemptible, check.Equals, true)
}
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index b0c7069cd..9c684bbe3 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -62,7 +62,7 @@ type InstanceType struct {
RAM int64
Scratch int64
Price float64
- Preemptable bool
+ Preemptible bool
}
// GetThisSystemNode returns a SystemNode for the node we're running
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index e71bcd5d0..5398d9d74 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -53,7 +53,7 @@ type RuntimeConstraints struct {
// such as Partitions
type SchedulingParameters struct {
Partitions []string `json:"partitions"`
- Preemptable bool `json:"preemptable"`
+ Preemptible bool `json:"preemptible"`
}
// ContainerList is an arvados#containerList resource.
diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb
index 42fd247f7..799aa430f 100644
--- a/services/api/app/models/container_request.rb
+++ b/services/api/app/models/container_request.rb
@@ -29,7 +29,7 @@ class ContainerRequest < ArvadosModel
before_validation :fill_field_defaults, :if => :new_record?
before_validation :validate_runtime_constraints
before_validation :set_container
- before_validation :set_default_preemptable_scheduling_parameter
+ before_validation :set_default_preemptible_scheduling_parameter
validates :command, :container_image, :output_path, :cwd, :presence => true
validates :output_ttl, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
validates :priority, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 1000 }
@@ -198,14 +198,14 @@ class ContainerRequest < ArvadosModel
end
end
- def set_default_preemptable_scheduling_parameter
+ def set_default_preemptible_scheduling_parameter
if self.state == Committed
- # If preemptable instances (eg: AWS Spot Instances) are allowed,
+ # If preemptible instances (eg: AWS Spot Instances) are allowed,
# ask them on child containers by default.
- if Rails.configuration.preemptable_instances and
+ if Rails.configuration.preemptible_instances and
!self.requesting_container_uuid.nil? and
- self.scheduling_parameters['preemptable'].nil?
- self.scheduling_parameters['preemptable'] = true
+ self.scheduling_parameters['preemptible'].nil?
+ self.scheduling_parameters['preemptible'] = true
end
end
end
@@ -236,8 +236,8 @@ class ContainerRequest < ArvadosModel
scheduling_parameters['partitions'].size)
errors.add :scheduling_parameters, "partitions must be an array of strings"
end
- if !Rails.configuration.preemptable_instances and scheduling_parameters['preemptable']
- errors.add :scheduling_parameters, "preemptable instances are not allowed"
+ if !Rails.configuration.preemptible_instances and scheduling_parameters['preemptible']
+ errors.add :scheduling_parameters, "preemptible instances are not allowed"
end
end
end
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 19b6f9b25..f51679135 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -289,10 +289,10 @@ common:
### Crunch, DNS & compute node management
###
- # Preemptable instance support (e.g. AWS Spot Instances)
- # When true, child containers will get created with the preemptable
+ # Preemptible instance support (e.g. AWS Spot Instances)
+ # When true, child containers will get created with the preemptible
# scheduling parameter parameter set.
- preemptable_instances: false
+ preemptible_instances: false
# Docker image to be used when none found in runtime_constraints of a job
default_docker_image_for_jobs: false
diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb
index b36ff06bb..8071e05ce 100644
--- a/services/api/test/unit/container_request_test.rb
+++ b/services/api/test/unit/container_request_test.rb
@@ -760,16 +760,16 @@ class ContainerRequestTest < ActiveSupport::TestCase
[
[false, ActiveRecord::RecordInvalid],
[true, nil],
- ].each do |preemptable_conf, expected|
- test "having Rails.configuration.preemptable_instances=#{preemptable_conf}, create preemptable container request and verify #{expected}" do
- sp = {"preemptable" => true}
+ ].each do |preemptible_conf, expected|
+ test "having Rails.configuration.preemptible_instances=#{preemptible_conf}, create preemptible container request and verify #{expected}" do
+ sp = {"preemptible" => true}
common_attrs = {cwd: "test",
priority: 1,
command: ["echo", "hello"],
output_path: "test",
scheduling_parameters: sp,
mounts: {"test" => {"kind" => "json"}}}
- Rails.configuration.preemptable_instances = preemptable_conf
+ Rails.configuration.preemptible_instances = preemptible_conf
set_user_from_auth :active
cr = create_minimal_req!(common_attrs)
@@ -790,15 +790,15 @@ class ContainerRequestTest < ActiveSupport::TestCase
'zzzzz-dz642-runningcontainr',
nil,
].each do |requesting_c|
- test "having preemptable instances active on the API server, a committed #{requesting_c.nil? ? 'non-':''}child CR should not ask for preemptable instance if parameter already set to false" do
+ test "having preemptible instances active on the API server, a committed #{requesting_c.nil? ? 'non-':''}child CR should not ask for preemptible instance if parameter already set to false" do
common_attrs = {cwd: "test",
priority: 1,
command: ["echo", "hello"],
output_path: "test",
- scheduling_parameters: {"preemptable" => false},
+ scheduling_parameters: {"preemptible" => false},
mounts: {"test" => {"kind" => "json"}}}
- Rails.configuration.preemptable_instances = true
+ Rails.configuration.preemptible_instances = true
set_user_from_auth :active
if requesting_c
@@ -813,7 +813,7 @@ class ContainerRequestTest < ActiveSupport::TestCase
cr.state = ContainerRequest::Committed
cr.save!
- assert_equal false, cr.scheduling_parameters['preemptable']
+ assert_equal false, cr.scheduling_parameters['preemptible']
end
end
@@ -822,15 +822,15 @@ class ContainerRequestTest < ActiveSupport::TestCase
[true, nil, nil],
[false, 'zzzzz-dz642-runningcontainr', nil],
[false, nil, nil],
- ].each do |preemptable_conf, requesting_c, schedule_preemptable|
- test "having Rails.configuration.preemptable_instances=#{preemptable_conf}, #{requesting_c.nil? ? 'non-':''}child CR should #{schedule_preemptable ? '':'not'} ask for preemptable instance by default" do
+ ].each do |preemptible_conf, requesting_c, schedule_preemptible|
+ test "having Rails.configuration.preemptible_instances=#{preemptible_conf}, #{requesting_c.nil? ? 'non-':''}child CR should #{schedule_preemptible ? '':'not'} ask for preemptible instance by default" do
common_attrs = {cwd: "test",
priority: 1,
command: ["echo", "hello"],
output_path: "test",
mounts: {"test" => {"kind" => "json"}}}
- Rails.configuration.preemptable_instances = preemptable_conf
+ Rails.configuration.preemptible_instances = preemptible_conf
set_user_from_auth :active
if requesting_c
@@ -845,7 +845,7 @@ class ContainerRequestTest < ActiveSupport::TestCase
cr.state = ContainerRequest::Committed
cr.save!
- assert_equal schedule_preemptable, cr.scheduling_parameters['preemptable']
+ assert_equal schedule_preemptible, cr.scheduling_parameters['preemptible']
end
end
diff --git a/services/nodemanager/arvnodeman/computenode/driver/ec2.py b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
index c453b91cc..2b1564279 100644
--- a/services/nodemanager/arvnodeman/computenode/driver/ec2.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/ec2.py
@@ -91,7 +91,7 @@ class ComputeNodeDriver(BaseComputeNodeDriver):
"VolumeSize": volsize,
"VolumeType": "gp2"
}}]
- if size.preemptable:
+ if size.preemptible:
# Request a Spot instance for this node
kw['ex_spot_market'] = True
return kw
diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index f9724a8fc..8c6757e51 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -151,15 +151,15 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
section_types = {
'instance_type': str,
'price': float,
- 'preemptable': bool,
+ 'preemptible': bool,
}
for sec_name in self.sections():
sec_words = sec_name.split(None, 2)
if sec_words[0] != 'Size':
continue
size_spec = self.get_section(sec_name, section_types, int)
- if 'preemptable' not in size_spec:
- size_spec['preemptable'] = False
+ if 'preemptible' not in size_spec:
+ size_spec['preemptible'] = False
if 'instance_type' not in size_spec:
# Assume instance type is Size name if missing
size_spec['instance_type'] = sec_words[1]
diff --git a/services/nodemanager/arvnodeman/jobqueue.py b/services/nodemanager/arvnodeman/jobqueue.py
index 99064b398..e91764474 100644
--- a/services/nodemanager/arvnodeman/jobqueue.py
+++ b/services/nodemanager/arvnodeman/jobqueue.py
@@ -38,7 +38,7 @@ class ServerCalculator(object):
self.cores = 0
self.bandwidth = 0
self.price = 9999999
- self.preemptable = False
+ self.preemptible = False
self.extra = {}
def meets_constraints(self, **kwargs):
@@ -58,7 +58,7 @@ class ServerCalculator(object):
self.disk = 0
self.scratch = self.disk * 1000
self.ram = int(self.ram * node_mem_scaling)
- self.preemptable = False
+ self.preemptible = False
for name, override in kwargs.iteritems():
if name == 'instance_type': continue
if not hasattr(self, name):
diff --git a/services/nodemanager/doc/ec2.example.cfg b/services/nodemanager/doc/ec2.example.cfg
index a1fa2dc32..117f9b224 100644
--- a/services/nodemanager/doc/ec2.example.cfg
+++ b/services/nodemanager/doc/ec2.example.cfg
@@ -169,12 +169,24 @@ security_groups = idstring1, idstring2
# You may also want to define the amount of scratch space (expressed
# in GB) for Crunch jobs. You can also override Amazon's provided
# data fields (such as price per hour) by setting them here.
+#
+# Additionally, you can ask for a preemptible instance (AWS's spot instance)
+# by adding the appropriate boolean configuration flag. If you want to have
+# both spot & reserved versions of the same size, you can do so by renaming
+# the Size section and specifying the instance type inside it.
[Size m4.large]
cores = 2
price = 0.126
scratch = 100
+[Size m4.large.spot]
+instance_type = m4.large
+preemptible = true
+cores = 2
+price = 0.126
+scratch = 100
+
[Size m4.xlarge]
cores = 4
price = 0.252
diff --git a/services/nodemanager/tests/test_computenode_driver_ec2.py b/services/nodemanager/tests/test_computenode_driver_ec2.py
index 9442a8c24..520c0dc0c 100644
--- a/services/nodemanager/tests/test_computenode_driver_ec2.py
+++ b/services/nodemanager/tests/test_computenode_driver_ec2.py
@@ -73,10 +73,10 @@ class EC2ComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
create_method.call_args[1].get('ex_metadata', {'arg': 'missing'}).items()
)
- def test_create_preemptable_instance(self):
+ def test_create_preemptible_instance(self):
arv_node = testutil.arvados_node_mock()
driver = self.new_driver()
- driver.create_node(testutil.MockSize(1, preemptable=True), arv_node)
+ driver.create_node(testutil.MockSize(1, preemptible=True), arv_node)
create_method = self.driver_mock().create_node
self.assertTrue(create_method.called)
self.assertEqual(
diff --git a/services/nodemanager/tests/test_config.py b/services/nodemanager/tests/test_config.py
index 9a48c7cda..8002b3b92 100644
--- a/services/nodemanager/tests/test_config.py
+++ b/services/nodemanager/tests/test_config.py
@@ -29,9 +29,9 @@ creds = dummy_creds
cores = 1
price = 0.8
-[Size 1.preemptable]
+[Size 1.preemptible]
instance_type = 1
-preemptable = true
+preemptible = true
cores = 1
price = 0.8
@@ -65,17 +65,17 @@ testlogger = INFO
self.assertEqual('Small', size.name)
self.assertEqual(1, kwargs['cores'])
self.assertEqual(0.8, kwargs['price'])
- # preemptable is False by default
- self.assertEqual(False, kwargs['preemptable'])
+ # preemptible is False by default
+ self.assertEqual(False, kwargs['preemptible'])
# instance_type == arvados node size id by default
self.assertEqual(kwargs['id'], kwargs['instance_type'])
- # Now retrieve the preemptable version
+ # Now retrieve the preemptible version
size, kwargs = sizes[1]
self.assertEqual('Small', size.name)
- self.assertEqual('1.preemptable', kwargs['id'])
+ self.assertEqual('1.preemptible', kwargs['id'])
self.assertEqual(1, kwargs['cores'])
self.assertEqual(0.8, kwargs['price'])
- self.assertEqual(True, kwargs['preemptable'])
+ self.assertEqual(True, kwargs['preemptible'])
self.assertEqual('1', kwargs['instance_type'])
diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py
index 2ec13c0b8..ee475efe7 100644
--- a/services/nodemanager/tests/testutil.py
+++ b/services/nodemanager/tests/testutil.py
@@ -78,7 +78,7 @@ class MockShutdownTimer(object):
class MockSize(object):
- def __init__(self, factor, preemptable=False):
+ def __init__(self, factor, preemptible=False):
self.id = 'z{}.test'.format(factor)
self.name = 'test size '+self.id
self.ram = 128 * factor
@@ -88,7 +88,7 @@ class MockSize(object):
self.price = float(factor)
self.extra = {}
self.real = self
- self.preemptable = preemptable
+ self.preemptible = preemptible
def __eq__(self, other):
return self.id == other.id
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list