[arvados] updated: 2.5.0-186-g475dc1027
git repository hosting
git at public.arvados.org
Tue Feb 21 21:59:53 UTC 2023
Summary of changes:
.../workbench/config/initializers/reload_config.rb | 4 +-
build/build-dev-docker-jobs-image.sh | 22 +-
doc/admin/spot-instances.html.textile.liquid | 10 +
go.mod | 31 +-
go.sum | 133 ++++---
lib/config/config.default.yml | 8 +-
lib/config/export.go | 2 +-
lib/crunchrun/crunchrun.go | 61 ++--
lib/crunchrun/crunchrun_test.go | 47 ++-
sdk/cwl/arvados_cwl/__init__.py | 9 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 25 +-
sdk/cwl/arvados_cwl/arvdocker.py | 32 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 392 +++++++++++++++++++--
sdk/cwl/arvados_cwl/context.py | 1 +
sdk/cwl/arvados_cwl/executor.py | 98 ++++--
sdk/cwl/arvados_cwl/fsaccess.py | 10 +-
sdk/cwl/arvados_cwl/runner.py | 192 +++++-----
sdk/cwl/setup.py | 4 +-
sdk/cwl/test_with_arvbox.sh | 11 +
sdk/cwl/tests/arvados-tests.yml | 2 +-
.../dmel_r6.16/WholeGenome/genome.dict | 0
.../dmel_r6.16/WholeGenome/genome.fa.fai | 0
.../Homo_sapiens/GRCh38.p2/WholeGenome/genome.dict | 0
.../GRCh38.p2/WholeGenome/genome.fa.fai | 0
.../collection_per_tool/collection_per_tool.cwl | 4 +-
.../collection_per_tool_wrapper.cwl | 2 +-
sdk/cwl/tests/test_container.py | 19 +-
sdk/cwl/tests/test_copy_deps.py | 72 ++--
sdk/cwl/tests/test_submit.py | 257 +++++---------
sdk/cwl/tests/wf/expect_upload_wrapper.cwl | 5 +-
sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl | 5 +-
sdk/cwl/tests/wf/runin-reqs-wf.cwl | 3 +-
sdk/cwl/tests/wf/runin-reqs-wf2.cwl | 3 +-
sdk/cwl/tests/wf/runin-reqs-wf3.cwl | 3 +-
sdk/cwl/tests/wf/runin-reqs-wf4.cwl | 3 +-
sdk/cwl/tests/wf/runin-reqs-wf5.cwl | 3 +-
sdk/cwl/tests/wf/secret_wf.cwl | 2 +-
sdk/cwl/tests/wf/submit_wf.cwl | 2 +-
sdk/cwl/tests/wf/submit_wf_no_reuse.cwl | 4 +-
sdk/cwl/tests/wf/submit_wf_process_properties.cwl | 2 +-
sdk/cwl/tests/wf/submit_wf_runner_resources.cwl | 2 +-
...ct_upload_wrapper.cwl => submit_wf_wrapper.cwl} | 30 +-
sdk/dev-jobs.dockerfile | 3 +
sdk/go/arvados/config.go | 2 +-
sdk/go/arvados/fs_collection.go | 59 +++-
sdk/go/arvados/fs_collection_test.go | 34 +-
sdk/python/arvados/util.py | 17 +
services/api/config/initializers/reload_config.rb | 4 +-
.../api/db/migrate/20221230155924_bigint_id.rb | 5 +
.../pillars/nginx_collections_configuration.sls | 5 +
.../aws/pillars/nginx_controller_configuration.sls | 5 +
.../aws/pillars/nginx_download_configuration.sls | 5 +
.../aws/pillars/nginx_keepproxy_configuration.sls | 5 +
.../aws/pillars/nginx_webshell_configuration.sls | 5 +
.../aws/pillars/nginx_websocket_configuration.sls | 5 +
.../aws/pillars/nginx_workbench2_configuration.sls | 5 +
.../aws/pillars/nginx_workbench_configuration.sls | 5 +
.../multi_host/aws/pillars/ssl_key_encrypted.sls | 11 +
.../multi_host/aws/states/custom_certs.sls | 10 +-
.../multi_host/aws/states/ssl_key_encrypted.sls | 71 ++++
tools/salt-install/installer.sh | 8 +-
.../local.params.example.multiple_hosts | 6 +
...l.params.example.single_host_multiple_hostnames | 5 +
...ocal.params.example.single_host_single_hostname | 5 +
tools/salt-install/provision.sh | 31 +-
.../salt-install/terraform/aws/services/locals.tf | 3 +
tools/salt-install/terraform/aws/services/main.tf | 44 ++-
.../salt-install/terraform/aws/services/outputs.tf | 4 +
.../terraform/aws/services/terraform.tfvars | 4 +
.../terraform/aws/services/variables.tf | 6 +
70 files changed, 1270 insertions(+), 617 deletions(-)
copy apps/workbench/app/mailers/.gitkeep => sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.dict (100%)
copy apps/workbench/app/mailers/.gitkeep => sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa.fai (100%)
copy apps/workbench/app/mailers/.gitkeep => sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.dict (100%)
copy apps/workbench/app/mailers/.gitkeep => sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa.fai (100%)
copy sdk/cwl/tests/wf/{expect_upload_wrapper.cwl => submit_wf_wrapper.cwl} (54%)
create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/ssl_key_encrypted.sls
create mode 100644 tools/salt-install/config_examples/multi_host/aws/states/ssl_key_encrypted.sls
via 475dc10274ca275966aa6eefc25b8932cc4f3957 (commit)
via 000e6ae927ca69b54d05572fa8073f003c442126 (commit)
via bfe0ea9b824dc057f07355a928ccb64ab68b6c57 (commit)
via b4792d835e27f7b5deaaff51adf648f395c3a999 (commit)
via c088a40d5a626c42d6779bca10c8a6f84a86f7bc (commit)
via fcfc961430e59fc19a0cee846c32bbe822463aa0 (commit)
via e03ebe77c257e44645c64c8cd71e5e7c115991ef (commit)
via 94dcdefbf8fe264daa28c5b15f68b304c683e390 (commit)
via bc3c1a3334cf5b39d73ac7f59311dccc93e8a00f (commit)
via 0d17360aaf96a28edddcb0c4394b1965a765088a (commit)
via a68ec8b7bed718acbbc55900e41847b1d319874c (commit)
via e10646b6cd9e178c284f14658d3a55149582db3b (commit)
via e9928bf0228070a287c086eb91cf1b278bded2db (commit)
via cffecc32488ab5c7ef44d1d855fb8c0e55d76b1f (commit)
via 2de2c96925cc3439305f16dced7f89bd9124853d (commit)
via 284f37a08fcdff15012b9f731000c57c1d7c56f1 (commit)
via df5588faa32a61d40968cf5c0ef50bdfb36985e3 (commit)
via d4093dea56055b8630b4ba4ca04ad0c891770db8 (commit)
via 7602078b5c8f56f6dddfd89a36cf65cb2e1c88d0 (commit)
via acb7d560e17143a8a82af26a774a6a334f868494 (commit)
via e1577a7fbba344f1c7d784d085d5f8b099bc28bf (commit)
via 7da7fa921c00de34babb3de13400f1d413042fe8 (commit)
via e51ba73431f40cfcb6a8f2c80bb5373f23609fcd (commit)
via aece35d20c892aa7b3df29b15ac0a8a167b0b3ab (commit)
via 1d8b1200de6bc1fc6df70f611446e1e6466244fd (commit)
via 511fc7559a6ad00468c9a452bdd1de63ad2c1f77 (commit)
via f3a1fcb306856fc904c7e8051ccb69ea85e5640f (commit)
via 1c50d29ea10fa90a379ee1d3fe7d8e05681d8622 (commit)
via cfba0af9859716e3a771dc03d205379ac6b8834f (commit)
via 1bd3fa5209c808445ee8fa3bcfde8e88b0fd32d7 (commit)
via ca210064364c6f45db1e2d6a936f21940a3cf03f (commit)
via 952458464e04ff17e2c8fc0ce89812912e09737c (commit)
via 4b1a3fdcb5065f7a34eeeda662e31f01d8224392 (commit)
via 4b2b6ca09aec9dfb9f3871bd202484b1ae4063b9 (commit)
via c4c57180c9ee9f79a1d272710aa7b8747d4d0c38 (commit)
via 083b86a4e748900bcc285cac8bfd2ecdd36679f6 (commit)
via 4e765b926f58f169bfca38268df9bdda84b55503 (commit)
via 62ede2cf371f51cbe8bac07c36ddf904e428262b (commit)
via 995df5b03c9304bc956b02e50225ad2e1f9dfd8d (commit)
via 9f42cb85807ebad098aaf6e0ab3218f763b712e2 (commit)
via a54c0f72656d883ae8f27d5074e35f60e61dce09 (commit)
via 37eb070f55b5ae0c622fb4bf0a946c9dd49b2752 (commit)
via 4f3739a4967959db6783408d8aad2137b9ebdab5 (commit)
via 6e45aef0eaea08b9fb3ba85783af8552757f8e6a (commit)
via a5bb9338a46680c3976544618d0a3fd5f0b65a96 (commit)
via fb27923f7ce22eddde0852d24d6c94323de41330 (commit)
via fcdf73b20710064a80807e3d8b579a11b629d193 (commit)
via cc095ef1f8c86b05a8e58f15c4086a2caa861ec5 (commit)
via 60da027b51ed0d29ae2aad263b9ebdf8cfb6f9b3 (commit)
via 5ed84016e9e9a210ecf26a01ad50d2d6a51694a5 (commit)
via 01f34b3dbf9d37a551f90c3025d42202b222713f (commit)
via d74058dabbd7bf7360ace73fd6b2874c6ae52809 (commit)
via d2464c017d342cda08842631f5aa6266f191fa41 (commit)
via 21c017bebaa9e8811170c113da77b9054def7117 (commit)
via 347ece2d7c00703487b87998eab06c5edf8796db (commit)
via 91dc5cddefb9d5f09c261b5097165517ea5fb33a (commit)
via 165ec76d4d36b03449fcd756cb03261f688c1a75 (commit)
via 07c70cc59ef9f17ee3baf98d69be76d9a6585235 (commit)
via 50bd5e3f71a1eb18c7721fc44a934413463baa7c (commit)
via 55e73facf08cc8651eb71529a2ab4e0657d3c870 (commit)
via b420ec626f8cb5cd7a8b4252dfc2be76ba3ba844 (commit)
via 62e7af59cbad5577423b844213be7b2f59709602 (commit)
via 9e1828a4263d7efc8fe7149c8c63ce0477551e8c (commit)
via 28b5ebc08c632a7d947883910b565ffdc6f85b68 (commit)
via 472742c1b4c2abd62325471094f25bbbb4770ede (commit)
via fec959159de7bb7bed944bd15868e11be0ecdcfa (commit)
via 58287d8af844ed344736db29590b2b06c58c97c8 (commit)
via 0920e90f2454f47f1e4cf97ebbceb69e8c443e59 (commit)
via 0eee268be6c2d47f310f49bf74bd356aaf5ad163 (commit)
via 889e9fa0fe0faf311770d6bf7639432853544d79 (commit)
from 9fa741f376df64f11b3af4e3fb7e4d30bcba795f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 475dc10274ca275966aa6eefc25b8932cc4f3957
Author: Tom Clegg <tom at curii.com>
Date: Tue Feb 21 16:58:02 2023 -0500
19961: Mention interrupt handling on admin doc page.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/admin/spot-instances.html.textile.liquid b/doc/admin/spot-instances.html.textile.liquid
index 703e70fb8..aa640b9fd 100644
--- a/doc/admin/spot-instances.html.textile.liquid
+++ b/doc/admin/spot-instances.html.textile.liquid
@@ -90,6 +90,16 @@ BaseHTTPError: AuthFailure.ServiceLinkedRoleCreationNotPermitted: The provided c
The account needs to have a service linked role created. This can be done by logging into the AWS account, go to _IAM Management_ → _Roles_ and create the @AWSServiceRoleForEC2Spot@ role by clicking on the @Create@ button, selecting @EC2@ service and @EC2 - Spot Instances@ use case.
+h3. Interruption notices
+
+When running a container on a spot instance, Arvados monitors the EC2 metadata endpoint for interruption notices. When an interruption notice is received, it is reported in a log entry in the @crunch-run.txt@ file as well as a @warning@ in the @runtime_status@ field of the affected container.
+
+Example excerpt from @crunch-run.txt@:
+
+<pre>
+2023-02-21T21:12:42.350719824Z Cloud provider indicates instance action "stop" scheduled for time "2023-02-21T21:14:42Z"
+</pre>
+
h2. Preemptible instances on Azure
For general information, see "Use Spot VMs in Azure":https://docs.microsoft.com/en-us/azure/virtual-machines/spot-vms.
commit 000e6ae927ca69b54d05572fa8073f003c442126
Author: Tom Clegg <tom at curii.com>
Date: Tue Feb 21 16:14:04 2023 -0500
19961: Reuse metadata token within expected TTL, adjust retry logic.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index ea4851218..a9c65cca4 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -1203,6 +1203,8 @@ func (runner *ContainerRunner) updateLogs() {
var spotInterruptionCheckInterval = 5 * time.Second
var ec2MetadataBaseURL = "http://169.254.169.254"
+const ec2TokenTTL = time.Second * 21600
+
func (runner *ContainerRunner) checkSpotInterruptionNotices() {
type ec2metadata struct {
Action string `json:"action"`
@@ -1210,38 +1212,47 @@ func (runner *ContainerRunner) checkSpotInterruptionNotices() {
}
runner.CrunchLog.Printf("Checking for spot interruptions every %v using instance metadata at %s", spotInterruptionCheckInterval, ec2MetadataBaseURL)
var metadata ec2metadata
+ var token string
+ var tokenExp time.Time
check := func() error {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute))
defer cancel()
- req, err := http.NewRequestWithContext(ctx, http.MethodPut, ec2MetadataBaseURL+"/latest/api/token", nil)
- if err != nil {
- return err
- }
- req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "21600")
- resp, err := http.DefaultClient.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- return fmt.Errorf("%s", resp.Status)
- }
- token, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return err
+ if token == "" || tokenExp.Sub(time.Now()) < time.Minute {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPut, ec2MetadataBaseURL+"/latest/api/token", nil)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", fmt.Sprintf("%d", int(ec2TokenTTL/time.Second)))
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("%s", resp.Status)
+ }
+ newtoken, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return err
+ }
+ token = strings.TrimSpace(string(newtoken))
+ tokenExp = time.Now().Add(ec2TokenTTL)
}
- req, err = http.NewRequestWithContext(ctx, http.MethodGet, ec2MetadataBaseURL+"/latest/meta-data/spot/instance-action", nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, ec2MetadataBaseURL+"/latest/meta-data/spot/instance-action", nil)
if err != nil {
return err
}
- req.Header.Set("X-aws-ec2-metadata-token", strings.TrimSpace(string(token)))
- resp, err = http.DefaultClient.Do(req)
+ req.Header.Set("X-aws-ec2-metadata-token", token)
+ resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
metadata = ec2metadata{}
- if resp.StatusCode == http.StatusNotFound {
+ switch resp.StatusCode {
+ case http.StatusOK:
+ break
+ case http.StatusNotFound:
// "If Amazon EC2 is not preparing to stop or
// terminate the instance, or if you
// terminated the instance yourself,
@@ -1249,7 +1260,10 @@ func (runner *ContainerRunner) checkSpotInterruptionNotices() {
// instance metadata and you receive an HTTP
// 404 error when you try to retrieve it."
return nil
- } else if resp.StatusCode != http.StatusOK {
+ case http.StatusUnauthorized:
+ token = ""
+ return fmt.Errorf("%s", resp.Status)
+ default:
return fmt.Errorf("%s", resp.Status)
}
err = json.NewDecoder(resp.Body).Decode(&metadata)
@@ -1265,12 +1279,13 @@ func (runner *ContainerRunner) checkSpotInterruptionNotices() {
if err != nil {
runner.CrunchLog.Printf("Error checking spot interruptions: %s", err)
failures++
- if failures > 3 {
- runner.CrunchLog.Printf("Giving up on checking spot interruptions after too many errors")
+ if failures > 5 {
+ runner.CrunchLog.Printf("Giving up on checking spot interruptions after too many consecutive failures")
return
}
continue
}
+ failures = 0
if metadata != lastmetadata {
lastmetadata = metadata
text := fmt.Sprintf("Cloud provider indicates instance action %q scheduled for time %q", metadata.Action, metadata.Time.UTC().Format(time.RFC3339))
diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go
index 786f9410a..5b4c6827b 100644
--- a/lib/crunchrun/crunchrun_test.go
+++ b/lib/crunchrun/crunchrun_test.go
@@ -13,6 +13,7 @@ import (
"io"
"io/ioutil"
"log"
+ "math/rand"
"net/http"
"net/http/httptest"
"os"
@@ -21,6 +22,7 @@ import (
"runtime/pprof"
"strings"
"sync"
+ "sync/atomic"
"syscall"
"testing"
"time"
@@ -777,38 +779,50 @@ func (s *TestSuite) TestRunAlreadyRunning(c *C) {
c.Check(ran, Equals, false)
}
-func (s *TestSuite) TestSpotInterruptionNotice(c *C) {
- var failedOnce bool
- var stoptime time.Time
- token := "fake-ec2-metadata-token"
- stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if !failedOnce {
+func ec2MetadataServerStub(c *C, token *string, failureRate float64, stoptime *atomic.Value) *httptest.Server {
+ failedOnce := false
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if !failedOnce || rand.Float64() < failureRate {
w.WriteHeader(http.StatusServiceUnavailable)
failedOnce = true
return
}
switch r.URL.Path {
case "/latest/api/token":
- fmt.Fprintln(w, token)
+ fmt.Fprintln(w, *token)
case "/latest/meta-data/spot/instance-action":
- if r.Header.Get("X-aws-ec2-metadata-token") != token {
+ if r.Header.Get("X-aws-ec2-metadata-token") != *token {
w.WriteHeader(http.StatusUnauthorized)
- } else if stoptime.IsZero() {
+ } else if t, _ := stoptime.Load().(time.Time); t.IsZero() {
w.WriteHeader(http.StatusNotFound)
} else {
- fmt.Fprintf(w, `{"action":"stop","time":"%s"}`, stoptime.Format(time.RFC3339))
+ fmt.Fprintf(w, `{"action":"stop","time":"%s"}`, t.Format(time.RFC3339))
}
default:
w.WriteHeader(http.StatusNotFound)
}
}))
+}
+
+func (s *TestSuite) TestSpotInterruptionNotice(c *C) {
+ s.testSpotInterruptionNotice(c, 0.1)
+}
+
+func (s *TestSuite) TestSpotInterruptionNoticeNotAvailable(c *C) {
+ s.testSpotInterruptionNotice(c, 1)
+}
+
+func (s *TestSuite) testSpotInterruptionNotice(c *C, failureRate float64) {
+ var stoptime atomic.Value
+ token := "fake-ec2-metadata-token"
+ stub := ec2MetadataServerStub(c, &token, failureRate, &stoptime)
defer stub.Close()
defer func(i time.Duration, u string) {
spotInterruptionCheckInterval = i
ec2MetadataBaseURL = u
}(spotInterruptionCheckInterval, ec2MetadataBaseURL)
- spotInterruptionCheckInterval = time.Second / 4
+ spotInterruptionCheckInterval = time.Second / 8
ec2MetadataBaseURL = stub.URL
go s.runner.checkSpotInterruptionNotices()
@@ -824,13 +838,18 @@ func (s *TestSuite) TestSpotInterruptionNotice(c *C) {
"state": "Locked"
}`, nil, func() int {
time.Sleep(time.Second)
- stoptime = time.Now().Add(time.Minute).UTC()
+ stoptime.Store(time.Now().Add(time.Minute).UTC())
+ token = "different-fake-ec2-metadata-token"
time.Sleep(time.Second)
return 0
})
- c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Checking for spot interruptions every 250ms using instance metadata at http://.*`)
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Checking for spot interruptions every 125ms using instance metadata at http://.*`)
c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Error checking spot interruptions: 503 Service Unavailable.*`)
- c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Cloud provider indicates instance action "stop" scheduled for time "`+stoptime.Format(time.RFC3339)+`".*`)
+ if failureRate == 1 {
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Giving up on checking spot interruptions after too many consecutive failures.*`)
+ } else {
+ c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Cloud provider indicates instance action "stop" scheduled for time "`+stoptime.Load().(time.Time).Format(time.RFC3339)+`".*`)
+ }
}
func (s *TestSuite) TestRunTimeExceeded(c *C) {
commit bfe0ea9b824dc057f07355a928ccb64ab68b6c57
Merge: 9fa741f37 b4792d835
Author: Tom Clegg <tom at curii.com>
Date: Tue Feb 21 15:45:12 2023 -0500
19961: Merge branch 'main'
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list