[ARVADOS] updated: 1.3.0-1165-g4aa3a732c
Git user
git at public.curoverse.com
Wed Jun 26 20:17:13 UTC 2019
Summary of changes:
.../app/assets/javascripts/components/edit_tags.js | 51 ++++++++++++-
doc/_config.yml | 5 +-
...min_list_collections_without_property_py.liquid | 15 ++++
...property_to_collections_under_project_py.liquid | 36 +++++++++
.../_admin_update_collection_property_py.liquid | 22 ++++++
doc/_includes/_install_compute_docker.liquid | 2 +-
doc/_includes/_install_postgres_database.liquid | 2 +-
doc/_includes/_mount_types.liquid | 2 +-
doc/_includes/_navbar_top.liquid | 2 +-
doc/admin/cloudtest.html.textile.liquid | 71 ++++++++++++++++++
...llection-managed-properties.html.textile.liquid | 85 ++++++++++++++++++++++
doc/admin/upgrading.html.textile.liquid | 27 ++++++-
.../install-dispatch-cloud.html.textile.liquid | 29 +++++++-
doc/install/install-postgresql.html.textile.liquid | 2 +-
doc/sdk/index.html.textile.liquid | 2 +-
.../collection-versioning.html.textile.liquid | 2 +-
lib/cloud/cloudtest/tester.go | 34 ++++++---
lib/config/config.default.yml | 14 ++++
lib/config/export.go | 3 +
lib/config/generated_config.go | 14 ++++
lib/controller/federation.go | 2 +-
lib/dispatchcloud/dispatcher.go | 15 ++++
lib/dispatchcloud/dispatcher_test.go | 1 +
lib/dispatchcloud/driver.go | 6 +-
lib/dispatchcloud/scheduler/interfaces.go | 3 +-
lib/dispatchcloud/scheduler/run_queue_test.go | 5 +-
lib/dispatchcloud/scheduler/sync.go | 1 +
lib/dispatchcloud/worker/pool.go | 35 ++++++---
sdk/cwl/arvados_cwl/executor.py | 15 ++--
sdk/cwl/arvados_cwl/pathmapper.py | 20 +++--
sdk/cwl/arvados_cwl/runner.py | 5 +-
...13976-keepref-wf.cwl => 15295-bad-keep-ref.cwl} | 7 +-
sdk/cwl/tests/arvados-tests.yml | 6 ++
sdk/cwl/tests/badkeep.yml | 3 +
sdk/go/arvados/config.go | 7 +-
.../src/main/java/org/arvados/sdk/Arvados.java | 2 +-
.../arvados/v1/containers_controller.rb | 3 +-
services/api/app/models/arvados_model.rb | 12 +++
services/api/app/models/collection.rb | 38 ++++++++++
services/api/app/models/container.rb | 61 ++++++++--------
.../arvados/v1/containers_controller_test.rb | 2 +-
services/api/test/unit/collection_test.rb | 61 ++++++++++++++++
services/keep-balance/balance_test.go | 3 +-
.../lib/arvbox/docker/runit-docker/runit-docker.c | 2 +-
44 files changed, 635 insertions(+), 100 deletions(-)
create mode 100644 doc/_includes/_admin_list_collections_without_property_py.liquid
create mode 100644 doc/_includes/_admin_set_property_to_collections_under_project_py.liquid
create mode 100644 doc/_includes/_admin_update_collection_property_py.liquid
create mode 100644 doc/admin/cloudtest.html.textile.liquid
create mode 100644 doc/admin/collection-managed-properties.html.textile.liquid
copy sdk/cwl/tests/{13976-keepref-wf.cwl => 15295-bad-keep-ref.cwl} (54%)
create mode 100644 sdk/cwl/tests/badkeep.yml
via 4aa3a732cac495fb384a521b2046f6b7f09dd13d (commit)
via 222e7c73cded78e392a4007aded4d01113149b85 (commit)
via 10d70a1c08984a699ac3f6b893fe6d2141c5ad9e (commit)
via 7eb7c12b0aabda5c937922a9696200ed13b5f51f (commit)
via cadfd226ac8911d34d7dc66ea75ac977ec1319d8 (commit)
via f0475ef9a07db9e587401e8826d05bb3c030d8cb (commit)
via 3d8fc8f64b7bd331a159ba2eee16618c94000348 (commit)
via 9ceb2df8ffaabceba72dbcc9be92cc4652565646 (commit)
via 566e908c800ba66fe4946523fee76f7047b4c189 (commit)
via e16542e3cb4fddf05c407cb013c9e1573eb9c289 (commit)
via 1519b9e7e458a6fe286a214ee27ab07c3be6faf1 (commit)
via ecdc08f29a97a6127cd36c71cc56683f09907289 (commit)
via f5764a05d616e8d3c55b94503d5a1f789ac66ea7 (commit)
via 067bb74eb31914bb9735e0b847bb91ba2f46fca5 (commit)
via 2f4429e623199b9820d2296cf3788300caa7394b (commit)
via f1eabfdd91a3897234d395a1ad2046746dd7d3ff (commit)
via e6eb33290995c821fe6c9c1f2fcd0d543d9b4580 (commit)
via 8b1a2a30f34ac6baef507858828997e7c87a28d8 (commit)
via 000158113f2c571ebac30d854b712d87c91e41cf (commit)
via 5acd68562b23c0293140a9de6443855b6612266b (commit)
via a1c4ece152e1e992c18f0cb368528cd2ac843b07 (commit)
via 49717fb59156c2b276ccc2fde0b9f2de71e812a6 (commit)
via ecc5385565685bb5a4c4b2e1af3afb3ffea30937 (commit)
via a981ba7008866165a3941707ba2a98df34c424e0 (commit)
via ee655972c729a727b56e1dd302a025dd96ad1aa7 (commit)
via d5628158b1bebe822a4b8b160e0edecce364077b (commit)
via 2870aa32d3ae875aac19af2da1b2297b9e57689f (commit)
via b441daaf99f0596170a083d798ba3276f3dbe565 (commit)
via 282a705c0cc2841182f8b670f6696b681ab12779 (commit)
via b64d59746117dffccb3dc62d8f47504a91247c5d (commit)
via 58f83049f6170ae830b578c42d1d8e997ef6b797 (commit)
via 5a65708195cd5d0c6a588fab96ae441f3cf0bd04 (commit)
via 654560f15c684635f2331363f04ad4cb3e9663d6 (commit)
via ead5147bf16393d723ed864eeb3e0d3f24238709 (commit)
via 2cf4ea4fe0e1eda66d510983bf27fb9836af5ff4 (commit)
via 8f9ba1d6405c397c40de94e155cea0c84120ff0c (commit)
via ed55da61a1516914d199ea149bfd227b90107ac9 (commit)
via 282d7a0c9d8279fa0a1293573e4313c8223be1b5 (commit)
via ec1bcd246ad1097a502d02a9f0239d7bbed6d958 (commit)
from e61d0c92e9f52c8db5e65026a551f62918aad1b4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4aa3a732cac495fb384a521b2046f6b7f09dd13d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jun 26 16:17:02 2019 -0400
15026: Add cloudtest page to admin docs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/doc/_config.yml b/doc/_config.yml
index 20a2085c1..36e8c64b8 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -180,6 +180,7 @@ navbar:
- Cloud:
- admin/storage-classes.html.textile.liquid
- admin/spot-instances.html.textile.liquid
+ - admin/cloudtest.html.textile.liquid
- Data Management:
- admin/collection-versioning.html.textile.liquid
- admin/collection-managed-properties.html.textile.liquid
diff --git a/doc/admin/cloudtest.html.textile.liquid b/doc/admin/cloudtest.html.textile.liquid
new file mode 100644
index 000000000..02549c3f5
--- /dev/null
+++ b/doc/admin/cloudtest.html.textile.liquid
@@ -0,0 +1,71 @@
+---
+layout: default
+navsection: admin
+title: Testing cloud configuration
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The @arvados-server@ package includes a @cloudtest@ tool that checks compatibility between your Arvados configuration, your cloud driver, your cloud provider's API, your cloud provider's VM instances, and the worker image you use with the *experimental* "cloud dispatcher":../install/install-dispatch-cloud.html.
+
+ at arvados-server cloudtest@ performs the following steps:
+# Create a new instance
+# Wait for it to finish booting
+# Run a shell command on the new instance (optional)
+# Pause while you log in to the new instance and do other tests yourself (optional)
+# Shut down the instance
+
+This is an easy way to expose problems like these:
+* Configured cloud credentials don't work
+* Configured image types don't work
+* Configured driver is not compatible with your cloud API/region
+* Newly created instances are not usable due to a network problem or misconfiguration
+* Newly created instances do not accept the configured SSH private key
+* Selected machine image does not boot properly
+* Selected machine image is incompatible with some instance types
+* Driver has bugs
+
+h2. Typical uses
+
+Before bringing up the @arvados-dispatch-cloud@ service for the first time, we recommend running @cloudtest@ to check your configuration:
+
+<pre>
+$ <span class="userinput">arvados-server cloudtest -command "crunch-run --list"</span>
+</pre>
+
+Before updating your configuration to use a new VM image, we recommend running @cloudtest@ with the new image:
+
+<notextile><pre>
+$ <span class="userinput">arvados-server cloudtest -image-id <b>new_image_id</b> -command "crunch-run --list"</span>
+</pre></notextile>
+
+After adding an instance type to your configuration, we recommend running @cloudtest@ with the new instance type:
+
+<notextile><pre>
+$ <span class="userinput">arvados-server cloudtest -instance-type <b>new_instance_type_name</b></span>
+</pre></notextile>
+
+For a full list of options, use the @-help@ flag:
+
+<pre>
+$ <span class="userinput">arvados-server cloudtest -help</span>
+Usage:
+ -command string
+ Run an interactive shell command on the test instance when it boots
+ -config file
+ Site configuration file (default "/etc/arvados/config.yml")
+ -destroy-existing
+ Destroy any existing instances tagged with our InstanceSetID, instead of erroring out
+ -image-id string
+ Image ID to use when creating the test instance (if empty, use cluster config)
+ -instance-set-id value
+ InstanceSetID tag value to use on the test instance (default "cloudtest-user at hostname.example")
+ -instance-type string
+ Instance type to create (if empty, use cheapest type in config)
+ -pause-before-destroy
+ Prompt and wait before destroying the test instance
+</pre>
diff --git a/doc/install/install-dispatch-cloud.html.textile.liquid b/doc/install/install-dispatch-cloud.html.textile.liquid
index bc3be8f1d..d72909077 100644
--- a/doc/install/install-dispatch-cloud.html.textile.liquid
+++ b/doc/install/install-dispatch-cloud.html.textile.liquid
@@ -153,10 +153,37 @@ Minimal configuration example for Azure:
</code></pre>
</notextile>
-h2. Install the dispatcher
+h2. Test your configuration
First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/install-manual-prerequisites.html#repos.
+Next, install the arvados-server package.
+
+On Red Hat-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo yum install arvados-server</span>
+</code></pre>
+</notextile>
+
+On Debian-based systems:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo apt-get install arvados-server</span>
+</code></pre>
+</notextile>
+
+Run the @cloudtest@ tool to verify that your configuration works. This creates a new cloud VM, confirms that it boots correctly and accepts your configured SSH private key, and shuts it down.
+
+<notextile>
+<pre><code>~$ <span class="userinput">arvados-server cloudtest && echo "OK!"</span>
+</code></pre>
+</notextile>
+
+Refer to the "cloudtest tool documentation":../admin/cloudtest.html for more information.
+
+h2. Install the dispatcher
+
On Red Hat-based systems:
<notextile>
commit 222e7c73cded78e392a4007aded4d01113149b85
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jun 26 15:19:08 2019 -0400
15026: Log duration of each list/create/destroy/exec operation.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/cloud/cloudtest/tester.go b/lib/cloud/cloudtest/tester.go
index dc5a4738a..e0a6a3597 100644
--- a/lib/cloud/cloudtest/tester.go
+++ b/lib/cloud/cloudtest/tester.go
@@ -92,7 +92,9 @@ func (t *tester) Run() bool {
foundExisting = true
if t.DestroyExisting {
lgr.Info("destroying existing instance with our InstanceSetID")
+ t0 := time.Now()
err := i.Destroy()
+ lgr := lgr.WithField("Duration", time.Since(t0))
if err != nil {
lgr.WithError(err).Error("error destroying existing instance")
} else {
@@ -133,13 +135,15 @@ func (t *tester) Run() bool {
"Tags": tags,
"InitCommand": initCommand,
}).Info("creating instance")
+ t0 := time.Now()
inst, err := t.is.Create(t.InstanceType, t.ImageID, tags, initCommand, t.SSHKey.PublicKey())
+ lgrC := t.Logger.WithField("Duration", time.Since(t0))
if err != nil {
// Create() might have failed due to a bug or network
// error even though the creation was successful, so
// it's safer to wait a bit for an instance to appear.
deferredError = true
- t.Logger.WithError(err).Error("error creating test instance")
+ lgrC.WithError(err).Error("error creating test instance")
t.Logger.WithField("Deadline", bootDeadline).Info("waiting for instance to appear anyway, in case the Create response was incorrect")
for err = t.refreshTestInstance(); err != nil; err = t.refreshTestInstance() {
if time.Now().After(bootDeadline) {
@@ -154,7 +158,7 @@ func (t *tester) Run() bool {
} else {
// Create() succeeded. Make sure the new instance
// appears right away in the Instances() list.
- t.Logger.WithField("Instance", inst.ID()).Info("created instance")
+ lgrC.WithField("Instance", inst.ID()).Info("created instance")
t.testInstance = &worker.TagVerifier{inst, t.secret}
t.showLoginInfo()
err = t.refreshTestInstance()
@@ -247,11 +251,15 @@ func (t *tester) refreshTestInstance() error {
func (t *tester) getInstances(tags cloud.InstanceTags) ([]cloud.Instance, error) {
var ret []cloud.Instance
t.Logger.WithField("FilterTags", tags).Info("getting instance list")
+ t0 := time.Now()
insts, err := t.is.Instances(tags)
if err != nil {
return nil, err
}
- t.Logger.WithField("N", len(insts)).Info("got instance list")
+ t.Logger.WithFields(logrus.Fields{
+ "Duration": time.Since(t0),
+ "N": len(insts),
+ }).Info("got instance list")
for _, i := range insts {
if i.Tags()[t.TagKeyPrefix+"InstanceSetID"] == string(t.SetID) {
ret = append(ret, i)
@@ -312,11 +320,13 @@ func (t *tester) runShellCommand(cmd string) error {
t.Logger.WithFields(logrus.Fields{
"Command": cmd,
}).Info("executing remote command")
+ t0 := time.Now()
stdout, stderr, err := t.executor.Execute(nil, cmd, nil)
lgr := t.Logger.WithFields(logrus.Fields{
- "Command": cmd,
- "stdout": string(stdout),
- "stderr": string(stderr),
+ "Duration": time.Since(t0),
+ "Command": cmd,
+ "stdout": string(stdout),
+ "stderr": string(stderr),
})
if err != nil {
lgr.WithError(err).Info("remote command failed")
@@ -333,19 +343,21 @@ func (t *tester) destroyTestInstance() bool {
}
for {
t.Logger.WithField("Instance", t.testInstance.ID()).Info("destroying instance")
+ t0 := time.Now()
err := t.testInstance.Destroy()
- if err != nil {
- t.Logger.WithError(err).WithField("Instance", t.testInstance.ID()).Error("error destroying instance")
+ lgr := t.Logger.WithField("Instance", t.testInstance.ID())
+ if lgr := lgr.WithField("Duration", time.Since(t0)); err != nil {
+ lgr.WithError(err).Error("error destroying instance")
} else {
- t.Logger.WithField("Instance", t.testInstance.ID()).Info("destroyed instance")
+ lgr.Info("destroyed instance")
}
err = t.refreshTestInstance()
if err == errTestInstanceNotFound {
- t.Logger.WithField("Instance", t.testInstance.ID()).Info("instance no longer appears in list")
+ lgr.Info("instance no longer appears in list")
t.testInstance = nil
return true
} else if err == nil {
- t.Logger.WithField("Instance", t.testInstance.ID()).Info("instance still exists after calling Destroy")
+ lgr.Info("instance still exists after calling Destroy")
t.sleepSyncInterval()
continue
} else {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list