[ARVADOS] updated: 1.1.4-343-gd946222
Git user
git at public.curoverse.com
Thu May 31 17:24:19 EDT 2018
Summary of changes:
sdk/cwl/arvados_cwl/__init__.py | 9 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 32 ++-
sdk/cwl/arvados_cwl/arvworkflow.py | 1 +
sdk/cwl/arvados_cwl/http.py | 141 ++++++++++
sdk/cwl/arvados_cwl/pathmapper.py | 6 +
sdk/cwl/arvados_version.py | 2 +-
sdk/cwl/setup.py | 4 +-
sdk/cwl/tests/test_container.py | 16 +-
sdk/cwl/tests/test_http.py | 286 +++++++++++++++++++++
sdk/cwl/tests/test_job.py | 6 +-
sdk/cwl/tests/test_submit.py | 29 ++-
services/keepstore/s3_volume.go | 4 +-
services/nodemanager/arvnodeman/config.py | 2 +-
services/nodemanager/arvnodeman/launcher.py | 2 +-
.../nodemanager/arvnodeman/test/fake_driver.py | 4 +-
services/nodemanager/tests/fake_azure.cfg.template | 8 +-
services/nodemanager/tests/fake_ec2.cfg.template | 8 +-
services/nodemanager/tests/fake_gce.cfg.template | 8 +-
services/nodemanager/tests/integration_test.py | 10 +-
19 files changed, 531 insertions(+), 47 deletions(-)
create mode 100644 sdk/cwl/arvados_cwl/http.py
create mode 100644 sdk/cwl/tests/test_http.py
via d9462226315eb4a479b6bb998aa51531062635b0 (commit)
via 6814a151d680775d9804d56582a94af90cee2ab8 (commit)
via 49bb6cb300e9f6a3862c4e17112e34da1226a3c7 (commit)
via f57c43c811ee8af08e7662ad567a6d8e8c18b781 (commit)
via 1bff2ab0181be31492c53351afc1c3c1e58ea05d (commit)
via 08f0ba557d67488933ddfa90030f80459efe7997 (commit)
via 7e9b9d420b58c22c41a713b9bfbfd0f5718abb1a (commit)
via 606c29cdbb9012ee99043a2da9f28b2cd302e5e4 (commit)
via c9abf73e0b57739a594aa1b4803726e612cf6313 (commit)
via 739b1b9ec3662f988ad09509bcc933ce5c23c4e8 (commit)
via 57e511e900aa1eb175fa7f308b09516ad83492b4 (commit)
via e4ea2e1dd1cb597b02d15bd7b9323705d6342f99 (commit)
via ddac86f6bae74398f58db75e383f5945c323a099 (commit)
via b3f04be24d0d2c75808dccd9e5923ee25371cb17 (commit)
via 6e17218f2fc39a828af4d3a19bdb29243739dafd (commit)
via 708a9f8e858f0791702083e3f429fcb3eea5af15 (commit)
via a7af6b95953992a7e9554b56604cf8248d5a2bf4 (commit)
via 9a52f3d07288f8a0073736a2a273ddaeb97ddc18 (commit)
via 6853ac7e518c5c6a888c5f5bd4edb27bdb564a2c (commit)
via 2600c9f3e0cbe1072c1bc4887aa5febe8906ddec (commit)
from 2f9761bd08293de1847d3dbd0ab16482eacc414d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d9462226315eb4a479b6bb998aa51531062635b0
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu May 31 17:17:33 2018 -0400
12061: Use shorter retry-after sleeps in integration test.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/test/fake_driver.py b/services/nodemanager/arvnodeman/test/fake_driver.py
index 5d03308..bf530f7 100644
--- a/services/nodemanager/arvnodeman/test/fake_driver.py
+++ b/services/nodemanager/arvnodeman/test/fake_driver.py
@@ -130,10 +130,10 @@ class RetryDriver(FakeDriver):
create_calls += 1
if create_calls < 2:
raise RateLimitReachedError(429, "Rate limit exceeded",
- headers={'retry-after': '12'})
+ headers={'retry-after': '2'})
elif create_calls < 3:
raise BaseHTTPError(429, "Rate limit exceeded",
- {'retry-after': '2'})
+ {'retry-after': '1'})
else:
return super(RetryDriver, self).create_node(name=name,
size=size,
diff --git a/services/nodemanager/tests/integration_test.py b/services/nodemanager/tests/integration_test.py
index fca61ec..4aa8c9e 100755
--- a/services/nodemanager/tests/integration_test.py
+++ b/services/nodemanager/tests/integration_test.py
@@ -433,8 +433,8 @@ def main():
# Actions (pattern -> action)
[
(r".*Daemon started", set_squeue),
- (r".*Rate limit exceeded - scheduling retry in 12 seconds", noop),
(r".*Rate limit exceeded - scheduling retry in 2 seconds", noop),
+ (r".*Rate limit exceeded - scheduling retry in 1 seconds", noop),
(r".*Cloud node (\S+) is now paired with Arvados node (\S+) with hostname (\S+)", noop),
],
# Checks (things that shouldn't happen)
commit 6814a151d680775d9804d56582a94af90cee2ab8
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu May 31 17:07:13 2018 -0400
12061: Don't wait around for test daemon to shutdown gracefully.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/nodemanager/tests/integration_test.py b/services/nodemanager/tests/integration_test.py
index 1699b57..fca61ec 100755
--- a/services/nodemanager/tests/integration_test.py
+++ b/services/nodemanager/tests/integration_test.py
@@ -216,7 +216,7 @@ def run_test(name, actions, checks, driver_class, jobs, provider):
if code != 0:
detail.error("Check failed")
if not terminated:
- p.terminate()
+ p.kill()
terminated = True
if terminated:
@@ -226,7 +226,7 @@ def run_test(name, actions, checks, driver_class, jobs, provider):
detail.error("Exceeded timeout with actions remaining: %s", actions)
code += 1
if not terminated:
- p.terminate()
+ p.kill()
terminated = True
k, v = actions[0]
@@ -237,11 +237,11 @@ def run_test(name, actions, checks, driver_class, jobs, provider):
code += v(g)
if code != 0:
detail.error("Action failed")
- p.terminate()
+ p.kill()
terminated = True
if not actions:
- p.terminate()
+ p.kill()
terminated = True
except KeyboardInterrupt:
p.kill()
commit 49bb6cb300e9f6a3862c4e17112e34da1226a3c7
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu May 31 17:06:32 2018 -0400
12061: Poll aggressively in integration tests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/nodemanager/tests/fake_azure.cfg.template b/services/nodemanager/tests/fake_azure.cfg.template
index 01f053c..a11a6d8 100644
--- a/services/nodemanager/tests/fake_azure.cfg.template
+++ b/services/nodemanager/tests/fake_azure.cfg.template
@@ -38,16 +38,16 @@ max_nodes = 8
max_total_price = 0
# Poll Azure nodes and Arvados for new information every N seconds.
-poll_time = 5
+poll_time = 0.5
# Polls have exponential backoff when services fail to respond.
# This is the longest time to wait between polls.
-max_poll_time = 300
+max_poll_time = 1
# If Node Manager can't succesfully poll a service for this long,
# it will never start or stop compute nodes, on the assumption that its
# information is too outdated.
-poll_stale_after = 600
+poll_stale_after = 1
# If Node Manager boots a cloud node, and it does not pair with an Arvados
# node before this long, assume that there was a cloud bootstrap failure and
@@ -115,7 +115,7 @@ driver_class = {driver_class}
# Azure bills by the minute, so it makes sense to agressively shut down idle
# nodes. Specify at least two windows. You can add as many as you need beyond
# that.
-shutdown_windows = 1, 999999
+shutdown_windows = 0.05, 999999
[Cloud Credentials]
# Use "azure account list" with the azure CLI to get these values.
diff --git a/services/nodemanager/tests/fake_ec2.cfg.template b/services/nodemanager/tests/fake_ec2.cfg.template
index 744d7f8..2bb7d0e 100644
--- a/services/nodemanager/tests/fake_ec2.cfg.template
+++ b/services/nodemanager/tests/fake_ec2.cfg.template
@@ -38,16 +38,16 @@ max_nodes = 8
max_total_price = 0
# Poll Azure nodes and Arvados for new information every N seconds.
-poll_time = 5
+poll_time = 0.5
# Polls have exponential backoff when services fail to respond.
# This is the longest time to wait between polls.
-max_poll_time = 300
+max_poll_time = 1
# If Node Manager can't succesfully poll a service for this long,
# it will never start or stop compute nodes, on the assumption that its
# information is too outdated.
-poll_stale_after = 600
+poll_stale_after = 1
# If Node Manager boots a cloud node, and it does not pair with an Arvados
# node before this long, assume that there was a cloud bootstrap failure and
@@ -115,7 +115,7 @@ driver_class = {driver_class}
# Azure bills by the minute, so it makes sense to agressively shut down idle
# nodes. Specify at least two windows. You can add as many as you need beyond
# that.
-shutdown_windows = 1, 999999
+shutdown_windows = 0.05, 999999
[Cloud Credentials]
diff --git a/services/nodemanager/tests/fake_gce.cfg.template b/services/nodemanager/tests/fake_gce.cfg.template
index 1c39ccf..11131ef 100644
--- a/services/nodemanager/tests/fake_gce.cfg.template
+++ b/services/nodemanager/tests/fake_gce.cfg.template
@@ -38,16 +38,16 @@ max_nodes = 8
max_total_price = 0
# Poll Azure nodes and Arvados for new information every N seconds.
-poll_time = 5
+poll_time = 0.5
# Polls have exponential backoff when services fail to respond.
# This is the longest time to wait between polls.
-max_poll_time = 300
+max_poll_time = 1
# If Node Manager can't succesfully poll a service for this long,
# it will never start or stop compute nodes, on the assumption that its
# information is too outdated.
-poll_stale_after = 600
+poll_stale_after = 1
# If Node Manager boots a cloud node, and it does not pair with an Arvados
# node before this long, assume that there was a cloud bootstrap failure and
@@ -115,7 +115,7 @@ driver_class = {driver_class}
# Azure bills by the minute, so it makes sense to agressively shut down idle
# nodes. Specify at least two windows. You can add as many as you need beyond
# that.
-shutdown_windows = 1, 999999
+shutdown_windows = 0.05, 999999
[Cloud Credentials]
key = 00000000-0000-0000-0000-000000000000
commit f57c43c811ee8af08e7662ad567a6d8e8c18b781
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu May 31 17:01:15 2018 -0400
12061: Allow sub-second poll times and sub-minute shutdown windows.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/nodemanager/arvnodeman/config.py b/services/nodemanager/arvnodeman/config.py
index e47f9fc..22ea8e3 100644
--- a/services/nodemanager/arvnodeman/config.py
+++ b/services/nodemanager/arvnodeman/config.py
@@ -154,5 +154,5 @@ class NodeManagerConfig(ConfigParser.SafeConfigParser):
return matching_sizes
def shutdown_windows(self):
- return [int(n)
+ return [float(n)
for n in self.get('Cloud', 'shutdown_windows').split(',')]
diff --git a/services/nodemanager/arvnodeman/launcher.py b/services/nodemanager/arvnodeman/launcher.py
index 888abf5..04a84c9 100644
--- a/services/nodemanager/arvnodeman/launcher.py
+++ b/services/nodemanager/arvnodeman/launcher.py
@@ -80,7 +80,7 @@ def build_server_calculator(config):
config.getfloat('Daemon', 'node_mem_scaling'))
def launch_pollers(config, server_calculator):
- poll_time = config.getint('Daemon', 'poll_time')
+ poll_time = config.getfloat('Daemon', 'poll_time')
max_poll_time = config.getint('Daemon', 'max_poll_time')
timer = TimedCallBackActor.start(poll_time / 10.0).tell_proxy()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list