[ARVADOS] updated: 2.1.0-2313-gb45a44c93
Git user
git at public.arvados.org
Thu Apr 14 19:09:27 UTC 2022
Summary of changes:
lib/config/config.default.yml | 26 ++++++++++++++++++--------
lib/crunchrun/crunchrun.go | 5 +++--
lib/crunchrun/integration_test.go | 27 +++++++++++++++++++++++++--
3 files changed, 46 insertions(+), 12 deletions(-)
via b45a44c93bcda724d862891eb2eed5666c8fd197 (commit)
via 817394edfcffb0f80e0a40e979e3027469bd9652 (commit)
from fb181ba27fd354e596d2216786ccee9a537bd0a3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b45a44c93bcda724d862891eb2eed5666c8fd197
Author: Tom Clegg <tom at curii.com>
Date: Thu Apr 14 15:08:56 2022 -0400
18992: Fixup and test log messages about config loading.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 6ac7a5f0c..474fbf4ad 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -2001,14 +2001,15 @@ func loadClusterConfigFile(path string, stderr io.Writer) *arvados.Cluster {
ldr.Path = path
cfg, err := ldr.Load()
if err != nil {
- fmt.Fprintf(stderr, "could not load config file %s: %s", path, err)
+ fmt.Fprintf(stderr, "could not load config file %s: %s\n", path, err)
return nil
}
cluster, err := cfg.GetCluster("")
if err != nil {
- fmt.Fprintf(stderr, "could not use config file %s: %s", path, err)
+ fmt.Fprintf(stderr, "could not use config file %s: %s\n", path, err)
return nil
}
+ fmt.Fprintf(stderr, "loaded config file %s\n", path)
return cluster
}
diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go
index ebd2726d0..0b139dd97 100644
--- a/lib/crunchrun/integration_test.go
+++ b/lib/crunchrun/integration_test.go
@@ -32,6 +32,7 @@ type integrationSuite struct {
stdin bytes.Buffer
stdout bytes.Buffer
stderr bytes.Buffer
+ args []string
cr arvados.ContainerRequest
client *arvados.Client
ac *arvadosclient.ArvadosClient
@@ -103,6 +104,7 @@ func (s *integrationSuite) TearDownSuite(c *C) {
func (s *integrationSuite) SetUpTest(c *C) {
os.Unsetenv("ARVADOS_KEEP_SERVICES")
s.engine = "docker"
+ s.args = nil
s.stdin = bytes.Buffer{}
s.stdout = bytes.Buffer{}
s.stderr = bytes.Buffer{}
@@ -219,6 +221,26 @@ func (s *integrationSuite) TestRunTrivialContainerWithLocalKeepstore(c *C) {
s.stdin.Reset()
s.testRunTrivialContainer(c)
c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*not starting a local keepstore process because a volume \(zzzzz-nyw5e-000000000000000\) uses AccessViaHosts\n.*`)
+
+ // Check that config read errors are logged
+ s.SetUpTest(c)
+ s.args = []string{"-config", c.MkDir() + "/config-error.yaml"}
+ s.stdin.Reset()
+ s.testRunTrivialContainer(c)
+ c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*could not load config file \Q`+s.args[1]+`\E:.* no such file or directory\n.*`)
+
+ s.SetUpTest(c)
+ s.args = []string{"-config", c.MkDir() + "/config-unreadable.yaml"}
+ s.stdin.Reset()
+ err := ioutil.WriteFile(s.args[1], []byte{}, 0)
+ c.Check(err, IsNil)
+ s.testRunTrivialContainer(c)
+ c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*could not load config file \Q`+s.args[1]+`\E:.* permission denied\n.*`)
+
+ s.SetUpTest(c)
+ s.stdin.Reset()
+ s.testRunTrivialContainer(c)
+ c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*loaded config file \Q`+os.Getenv("ARVADOS_CONFIG")+`\E\n.*`)
}
func (s *integrationSuite) testRunTrivialContainer(c *C) {
@@ -231,11 +253,12 @@ func (s *integrationSuite) testRunTrivialContainer(c *C) {
args := []string{
"-runtime-engine=" + s.engine,
"-enable-memory-limit=false",
- s.cr.ContainerUUID,
}
if s.stdin.Len() > 0 {
- args = append([]string{"-stdin-config=true"}, args...)
+ args = append(args, "-stdin-config=true")
}
+ args = append(args, s.args...)
+ args = append(args, s.cr.ContainerUUID)
code := command{}.RunCommand("crunch-run", args, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr))
c.Logf("\n===== stdout =====\n%s", s.stdout.String())
c.Logf("\n===== stderr =====\n%s", s.stderr.String())
commit 817394edfcffb0f80e0a40e979e3027469bd9652
Author: Tom Clegg <tom at curii.com>
Date: Thu Apr 14 14:41:57 2022 -0400
18992: Document how to use local keepstores with HPC dispatch.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 651238981..e60880c21 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -969,15 +969,25 @@ Clusters:
# A zero value disables this feature.
#
# In order for this feature to be activated, no volume may use
- # AccessViaHosts, and each volume must have Replication higher
- # than Collections.DefaultReplication. If these requirements are
- # not satisfied, the feature is disabled automatically
- # regardless of the value given here.
+ # AccessViaHosts, and no writable volume may have Replication
+ # lower than Collections.DefaultReplication. If these
+ # requirements are not satisfied, the feature is disabled
+ # automatically regardless of the value given here.
#
- # Note that when this configuration is enabled, the entire
- # cluster configuration file, including the system root token,
- # is copied to the worker node and held in memory for the
- # duration of the container.
+ # When an HPC dispatcher is in use (see SLURM and LSF sections),
+ # this feature depends on the operator to ensure an up-to-date
+ # cluster configuration file (/etc/arvados/config.yml) is
+ # available on all compute nodes. If it is missing or not
+ # readable by the crunch-run user, the feature will be disabled
+ # automatically. To read it from a different location, add a
+ # "-config=/path/to/config.yml" argument to
+ # CrunchRunArgumentsList above.
+ #
+ # When the cloud dispatcher is in use (see CloudVMs section) and
+ # this configuration is enabled, the entire cluster
+ # configuration file, including the system root token, is copied
+ # to the worker node and held in memory for the duration of the
+ # container.
LocalKeepBlobBuffersPerVCPU: 1
# When running a dedicated keepstore process for a container
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list