[ARVADOS] updated: 1.3.0-1598-gb144996ab
Git user
git at public.curoverse.com
Tue Sep 10 20:37:11 UTC 2019
Summary of changes:
.licenseignore | 1 +
.../configure-fs-storage.html.textile.liquid | 108 +++++++-----
doc/install/install-keepstore.html.textile.liquid | 185 ++++++---------------
lib/config/config.default.yml | 16 +-
lib/config/deprecated_keepstore.go | 3 +
lib/config/deprecated_keepstore_test.go | 29 +++-
lib/config/export.go | 6 +-
lib/config/generated_config.go | 16 +-
sdk/cwl/arvados_cwl/__init__.py | 3 +-
sdk/cwl/arvados_cwl/runner.py | 3 +-
sdk/cwl/setup.py | 4 +-
sdk/cwl/tests/test_submit.py | 36 ++++
sdk/cwl/tests/wf/feddemo | 1 +
sdk/cwl/tests/wf/revsort/revsort.cwl | 62 +++++++
sdk/cwl/tests/wf/revsort/revtool.cwl | 45 +++++
sdk/cwl/tests/wf/revsort/sorttool.cwl | 42 +++++
sdk/go/arvados/config.go | 13 +-
.../arvados/v1/keep_services_controller.rb | 18 +-
.../app/controllers/user_sessions_controller.rb | 96 ++++-------
.../api/app/models/api_client_authorization.rb | 154 ++++++++++-------
services/api/app/models/container_request.rb | 23 ++-
services/api/app/models/keep_service.rb | 42 +++++
services/api/app/models/user.rb | 87 +++++++++-
services/api/config/arvados_config.rb | 2 +
... 20190905151603_enforce_unique_identity_url.rb} | 4 +-
services/api/db/structure.sql | 10 +-
services/api/test/fixtures/users.yml | 8 +-
.../arvados/v1/keep_services_controller_test.rb | 30 ++++
.../functional/user_sessions_controller_test.rb | 31 +++-
services/api/test/integration/remote_user_test.rb | 132 ++++++++++-----
services/api/test/integration/users_test.rb | 28 ++++
services/api/test/unit/container_request_test.rb | 9 +-
services/api/test/unit/user_test.rb | 85 ++++++++++
services/login-sync/Gemfile.lock | 8 +-
vendor/vendor.json | 16 +-
35 files changed, 975 insertions(+), 381 deletions(-)
create mode 120000 sdk/cwl/tests/wf/feddemo
create mode 100644 sdk/cwl/tests/wf/revsort/revsort.cwl
create mode 100644 sdk/cwl/tests/wf/revsort/revtool.cwl
create mode 100644 sdk/cwl/tests/wf/revsort/sorttool.cwl
copy services/api/db/migrate/{20130107181109_add_uuid_to_collections.rb => 20190905151603_enforce_unique_identity_url.rb} (50%)
via b144996ab73b15233f027713faf3114f34985e72 (commit)
via 90d84d8578b760b493ac76b22c42bc284868bc0c (commit)
via 20a69e126c68236a3c73daab101e351346f8bfaf (commit)
via 04c702237cf2b545992ce7b1acad107a52118d72 (commit)
via 91bee39e7e5bcba0d5e418b532fc34949ffbefa8 (commit)
via 93086f7d095a33de76d7ad6353bbf3333178cf36 (commit)
via 6b43b14a374f217ddf5870c2ed9cd90152d8e6e2 (commit)
via 65705937e9c86a4ee557a32b776745e4ae8e88db (commit)
via 3ee696c6485008044b1a54c351d0035ef485305e (commit)
via 1c3c8f7fd2e1268b139e046fbd6a7093dd82222f (commit)
via e22d3dc998f55e3c21125b1a1be7240f89c23dd6 (commit)
via 9c656c55fe63a22b075223fe7f295e7e21e18b8c (commit)
via 168c5a9a50b93f736b15b7a6c56af900b90aab39 (commit)
via 7c3e13d4876a8e37feffee3cdaebc44b20b7b61b (commit)
via 934975a33a16ee8bf2ca854d263b98452856373f (commit)
via 2fd0a7680e075431baa61288f34bf400ccaae849 (commit)
via 068e72307f9ce91538e45cb461b99cf92d2f5666 (commit)
via 9ff2b6f56e4d0c071d2f436a78bc536c541eaf16 (commit)
via a3c2d73b2a3e87eab35853429f06195e15f1c972 (commit)
via 9902ba33a4006d8652e675f76b6d7e43a2446d14 (commit)
via a6f874e7de1026f9c60bc4769f919da031927ebe (commit)
via 636509feb523592f8e54f5fdbe52830efd105564 (commit)
via d6c4fc82452b6c8e7fe492a0e2a163a19477f95a (commit)
via 476f6188d78a8d7c60043b0dd5d22bfba045484c (commit)
via 5713754c574254f9e3650ac80bf8fdca235898f6 (commit)
via fb5b1feb7acdfafed1bcbe932ff90e95e129c2a3 (commit)
via 4653d82e57660b2f2d5cc9ce0960f44684b160ec (commit)
via 383122c1bc4a08772df618a415420ac3ea527051 (commit)
via f676243c0372529cdaa5821d4ebe9f9f59d0cd5a (commit)
via 0b471c74f2cc392a37aa4f8df8ed931bb5969236 (commit)
via b19eea5a71ff3fa6259df25bc0726bd1e152d89b (commit)
via fb7a094681db65daa2f28b50ca734dc27e9d6db7 (commit)
via d49b2b2d3c9ef513335adbc5785d252bf2fb68be (commit)
via 03ac6650c846556e8607e5a3379d03b7d44e62f9 (commit)
via 3d9b68358f757f1d3c94a63b641522735c387074 (commit)
from f04693da1811e670d4cbb981debeecf14d79137c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b144996ab73b15233f027713faf3114f34985e72
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Sep 10 16:37:05 2019 -0400
13647: Fixup docs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/doc/install/configure-fs-storage.html.textile.liquid b/doc/install/configure-fs-storage.html.textile.liquid
index 1e13190aa..2a3117ecb 100644
--- a/doc/install/configure-fs-storage.html.textile.liquid
+++ b/doc/install/configure-fs-storage.html.textile.liquid
@@ -1,7 +1,7 @@
---
layout: default
navsection: installguide
-title: Filesystem storage
+title: Configure filesystem storage
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
@@ -15,7 +15,7 @@ h2. Setting up filesystem mounts
Volumes are configured in the @Volumes@ section of the cluster configuration file. You may provide multiple volumes for a single keepstore process to manage multiple disks. Keepstore distributes blocks among volumes in round-robin fashion.
-Note that each volume has a UUID, like "zzzzz-nyw5e-0123456789abcde". You can assign these manually, but once assigned they should not be changed.
+{% include 'assign_volume_uuid' %}
Note that each volume has an AccessViaHosts section indicating that (for example) keep0's /mnt/local-disk directory is volume 0, while keep1's /mnt/local-disk directory is volume 1.
diff --git a/doc/install/install-keepstore.html.textile.liquid b/doc/install/install-keepstore.html.textile.liquid
index 1f724191e..92fd91eb0 100644
--- a/doc/install/install-keepstore.html.textile.liquid
+++ b/doc/install/install-keepstore.html.textile.liquid
@@ -69,7 +69,6 @@ Add or update the following sections of @/etc/arvados/config.yml@ as needed. Ref
MaxKeepBlockBuffers: 128
</code></pre>
</notextile>
-<pre>
h3. Notes on storage management
@@ -112,7 +111,7 @@ If your system uses systemd, the keepstore service should already be set up. Res
</code></pre>
</notextile>
-h3. Run keepstore as a supervised service
+h3(#runit). Start the service (option 2: runit)
Install runit to supervise the keepstore daemon. {% include 'install_runit' %}
@@ -126,11 +125,11 @@ GOGC=10 exec keepstore
</code></pre>
</notextile>
-h3. Set up additional servers
+h2. Set up additional servers
Repeat the above sections to prepare volumes and bring up supervised services on each Keepstore server you are setting up.
-h3. Restart the API server and controller
+h2. Restart the API server and controller
After adding all of your keepstore servers to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
@@ -138,7 +137,7 @@ After adding all of your keepstore servers to the Services section, make sure th
sudo systemctl restart nginx arvados-controller
</pre>
-h3(#testing). Testing keep
+h2(#testing). Testing keep
Install the "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
commit 90d84d8578b760b493ac76b22c42bc284868bc0c
Merge: 20a69e126 04c702237
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Sep 10 15:22:45 2019 -0400
13647: Merge branch 'master' into 13647-keepstore-config
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit 20a69e126c68236a3c73daab101e351346f8bfaf
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Sep 10 15:20:16 2019 -0400
13647: Update keepstore install docs, eliminate keep_services step.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/doc/install/configure-fs-storage.html.textile.liquid b/doc/install/configure-fs-storage.html.textile.liquid
index ddd54c3f0..1e13190aa 100644
--- a/doc/install/configure-fs-storage.html.textile.liquid
+++ b/doc/install/configure-fs-storage.html.textile.liquid
@@ -13,44 +13,68 @@ Keepstore can store data in local and network-attached POSIX filesystems.
h2. Setting up filesystem mounts
-Volumes are configured in the @Volumes@ section of the configuration file. You may provide multiple volumes for a single keepstore process to manage multiple disks. Keepstore distributes blocks among volumes in round-robin fashion.
-
-<pre>
-Volumes:
-- # The volume type, indicates this is a filesystem directory.
- Type: Directory
-
- # The directory that will be used as the backing store.
- Root: /mnt/local-disk
-
- # How much replication is performed by the underlying filesystem.
- # (for example, a network filesystem may provide its own replication).
- # This is used to inform replication decisions at the Keep layer.
- DirectoryReplication: 1
-
- # If true, do not accept write or trash operations, only reads.
- ReadOnly: false
-
- # When true, read and write operations (for whole 64MiB blocks) on
- # an individual volume will queued and issued serially. When
- # false, read and write operations will be issued concurrently.
- #
- # May improve throughput if you experience contention when there are
- # multiple requests to the same volume.
- #
- # When using SSDs, RAID, or a parallel network filesystem, you probably
- # don't want this.
- Serialize: false
-
- # Storage classes to associate with this volume. See "Storage
- # classes" in the "Admin" section of doc.arvados.org.
- StorageClasses: null
-
- # Example of a second volume section
-- DirectoryReplication: 2
- ReadOnly: false
- Root: /mnt/network-disk
- Serialize: false
- StorageClasses: null
- Type: Directory
-</pre>
+Volumes are configured in the @Volumes@ section of the cluster configuration file. You may provide multiple volumes for a single keepstore process to manage multiple disks. Keepstore distributes blocks among volumes in round-robin fashion.
+
+Note that each volume has a UUID, like "zzzzz-nyw5e-0123456789abcde". You can assign these manually, but once assigned they should not be changed.
+
+Note that each volume has an AccessViaHosts section indicating that (for example) keep0's /mnt/local-disk directory is volume 0, while keep1's /mnt/local-disk directory is volume 1.
+
+<notextile>
+<pre><code>Clusters:
+ <span class="userinput">uuid_prefix</span>:
+ Volumes:
+ <span class="userinput">uuid_prefix</span>-nyw5e-<span class="userinput">000000000000000</span>:
+ AccessViaHosts:
+ "http://keep0.zzzzz.example.com:25107/": {}
+ Driver: Directory
+ DriverParameters:
+ # The directory that will be used as the backing store.
+ Root: /mnt/local-disk
+
+ # When true, read and write operations (for whole 64MiB
+ # blocks) on an individual volume will queued and issued
+ # serially. When false, read and write operations will be
+ # issued concurrently.
+ #
+ # May improve throughput if you experience contention when
+ # there are multiple requests to the same volume.
+ #
+ # When using SSDs, RAID, or a shared network filesystem, you
+ # probably don't want this.
+ Serialize: false
+
+ # How much replication is performed by the underlying
+ # filesystem. (for example, a network filesystem may provide
+ # its own replication). This is used to inform replication
+ # decisions at the Keep layer.
+ Replication: 1
+
+ # If true, do not accept write or trash operations, only
+ # reads.
+ ReadOnly: false
+
+ # Storage classes to associate with this volume. See "Storage
+ # classes" in the "Admin" section of doc.arvados.org.
+ StorageClasses: null
+
+ <span class="userinput">uuid_prefix</span>-nyw5e-<span class="userinput">000000000000001</span>:
+ AccessViaHosts:
+ "http://keep1.zzzzz.example.com:25107/": {}
+ Driver: Directory
+ DriverParameters:
+ Root: /mnt/local-disk
+</code></pre></notextile>
+
+In the case of a network-attached filesystem accessible by all keepstore servers, the AccessViaHosts section is omitted.
+
+<notextile>
+<pre><code>Clusters:
+ <span class="userinput">uuid_prefix</span>:
+ Volumes:
+ # This volume is used by all keepstore servers.
+ <span class="userinput">uuid_prefix</span>-nyw5e-<span class="userinput">000000000000002</span>:
+ Driver: Directory
+ DriverParameters:
+ Root: /mnt/network-attached-filesystem
+ Replication: 2
+</code></pre></notextile>
diff --git a/doc/install/install-keepstore.html.textile.liquid b/doc/install/install-keepstore.html.textile.liquid
index 5044cc0c2..1f724191e 100644
--- a/doc/install/install-keepstore.html.textile.liquid
+++ b/doc/install/install-keepstore.html.textile.liquid
@@ -47,118 +47,29 @@ Verify that Keepstore is functional:
</code></pre>
</notextile>
-h3. Create config file
+h3. Create a superuser token
-By default, keepstore will look for its configuration file at @/etc/arvados/keepstore/keepstore.yml@
+If you haven't already done so, create a superuser token.
-You can override the configuration file location using the @-config@ command line option to keepstore.
+{% include 'create_superuser_token' %}
+
+h3. Update cluster config file
-The following is a sample configuration file:
+Add or update the following sections of @/etc/arvados/config.yml@ as needed. Refer to the examples and comments in the "default config.yml file":{{site.baseurl}}/admin/config.html for more information.
+<notextile>
+<pre><code>Clusters:
+ <span class="userinput">uuid_prefix</span>:
+ SystemRootToken: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+ Services:
+ Keepstore:
+ InternalURLs:
+ "http://<span class="userinput">keep0.uuid_prefix.your.domain</span>:25107/": {}
+ API:
+ MaxKeepBlockBuffers: 128
+</code></pre>
+</notextile>
<pre>
-# Duration for which new permission signatures (returned in PUT
-# responses) will be valid. This should be equal to the API
-# server's blob_signature_ttl configuration entry.
-BlobSignatureTTL: 336h0m0s
-
-# Local file containing the secret blob signing key (used to generate
-# and verify blob signatures). The contents of the key file must be
-# identical to the API server's blob_signing_key configuration entry.
-BlobSigningKeyFile: ""
-
-# Print extra debug logging
-Debug: false
-
-# Maximum number of concurrent block deletion operations (per
-# volume) when emptying trash. Default is 1.
-EmptyTrashWorkers: 1
-
-# Enable trash and delete features. If false, trash lists will be
-# accepted but blocks will not be trashed or deleted.
-# Keepstore does not delete data on its own. The keep-balance
-# service determines which blocks are candidates for deletion
-# and instructs the keepstore to move those blocks to the trash.
-EnableDelete: true
-
-# Local port to listen on. Can be 'address:port' or ':port', where
-# 'address' is a host IP address or name and 'port' is a port number
-# or name.
-Listen: :25107
-
-# Format of request/response and error logs: "json" or "text".
-LogFormat: json
-
-# The secret key that must be provided by monitoring services when
-# using the health check and metrics endpoints (/_health, /metrics).
-ManagementToken: xyzzy
-
-# Maximum RAM to use for data buffers, given in multiples of block
-# size (64 MiB). When this limit is reached, HTTP requests requiring
-# buffers (like GET and PUT) will wait for buffer space to be
-# released.
-#
-# It should be set such that MaxBuffers * 64MiB + 10% fits
-# comfortably in memory. On a host dedicated to running keepstore,
-# divide total memory by 88MiB to suggest a suitable value. For example,
-# if grep MemTotal /proc/meminfo reports MemTotal: 7125440 kB,
-# compute 7125440 / (88 * 1024)=79 and configure MaxBuffers: 79
-MaxBuffers: 128
-
-# Maximum concurrent requests. When this limit is reached, new
-# requests will receive 503 responses. Note: this limit does not
-# include idle connections from clients using HTTP keepalive, so it
-# does not strictly limit the number of concurrent connections. If
-# omitted or zero, the default is 2 * MaxBuffers.
-MaxRequests: 0
-
-# Path to write PID file during startup. This file is kept open and
-# locked with LOCK_EX until keepstore exits, so "fuser -k pidfile" is
-# one way to shut down. Exit immediately if there is an error
-# opening, locking, or writing the PID file.
-PIDFile: ""
-
-# Maximum number of concurrent pull operations. Default is 1, i.e.,
-# pull lists are processed serially. A pull operation copies a block
-# from another keepstore server.
-PullWorkers: 1
-
-# Honor read requests only if a valid signature is provided. This
-# should be true, except for development use and when migrating from
-# a very old version.
-RequireSignatures: true
-
-# Local file containing the Arvados API token used by keep-balance
-# or data manager. Delete, trash, and index requests are honored
-# only for this token.
-SystemAuthTokenFile: ""
-
-# Path to server certificate file in X509 format. Enables TLS mode.
-#
-# Example: /var/lib/acme/live/keep0.example.com/fullchain
-TLSCertificateFile: ""
-
-# Path to server key file in X509 format. Enables TLS mode.
-#
-# The key pair is read from disk during startup, and whenever SIGHUP
-# is received.
-#
-# Example: /var/lib/acme/live/keep0.example.com/privkey
-TLSKeyFile: ""
-
-# How often to check for (and delete) trashed blocks whose
-# TrashLifetime has expired.
-TrashCheckInterval: 24h0m0s
-
-# Time duration after a block is trashed during which it can be
-# recovered using an /untrash request.
-TrashLifetime: 336h0m0s
-
-# Maximum number of concurrent trash operations (moving a block to the
-# trash, or permanently deleting it) . Default is 1, i.e., trash lists
-# are processed serially. If individual trash operations have high
-# latency (eg some cloud platforms) you should increase this.
-TrashWorkers: 1
-</pre>
h3. Notes on storage management
@@ -178,6 +89,29 @@ Available storage volume types include POSIX filesystems and cloud object storag
* If you are using S3-compatible object storage (including Amazon S3, Google Cloud Storage, and Ceph RADOS), follow the setup instructions on "S3 Object Storage":configure-s3-object-storage.html
* If you are using Azure Blob Storage, follow the setup instructions on "Azure Blob Storage":configure-azure-blob-storage.html
+h2. Run keepstore as a supervised service
+
+h3. Start the service (option 1: systemd)
+
+If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
+
+If your system uses systemd, the keepstore service should already be set up. Restart it to read the updated configuration, and check its status:
+
+<notextile>
+<pre><code>~$ <span class="userinput">sudo systemctl restart keepstore</span>
+~$ <span class="userinput">sudo systemctl status keepstore</span>
+● keepstore.service - Arvados Keep Storage Daemon
+ Loaded: loaded (/etc/systemd/system/keepstore.service; enabled; vendor preset: enabled)
+ Active: active (running) since Tue 2019-09-10 14:16:29 UTC; 1s ago
+ Docs: https://doc.arvados.org/
+ Main PID: 25465 (keepstore)
+ Tasks: 9 (limit: 4915)
+ CGroup: /system.slice/keepstore.service
+ └─25465 /usr/bin/keepstore
+[...]
+</code></pre>
+</notextile>
+
h3. Run keepstore as a supervised service
Install runit to supervise the keepstore daemon. {% include 'install_runit' %}
@@ -188,7 +122,7 @@ Install this script as the run script @/etc/sv/keepstore/run@ for the keepstore
<pre><code>#!/bin/sh
exec 2>&1
-GOGC=10 exec keepstore -config /etc/arvados/keepstore/keepstore.yml
+GOGC=10 exec keepstore
</code></pre>
</notextile>
@@ -196,36 +130,14 @@ h3. Set up additional servers
Repeat the above sections to prepare volumes and bring up supervised services on each Keepstore server you are setting up.
-h3. Tell the API server about the Keepstore servers
-
-The API server needs to be informed about the presence of your Keepstore servers.
-
-First, if you don't already have an admin token, create a superuser token.
+h3. Restart the API server and controller
-{% include 'create_superuser_token' %}
-
-Configure your environment to run @arv@ using the output of create_superuser_token.rb:
+After adding all of your keepstore servers to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
<pre>
-export ARVADOS_API_HOST=zzzzz.example.com
-export ARVADOS_API_TOKEN=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
+sudo systemctl restart nginx arvados-controller
</pre>
-Use this command to register each keepstore server you have installed. Make sure to update the @service_host@ value.
-
-<notextile>
-<pre><code>~$ <span class="userinput">uuid_prefix=`arv --format=uuid user current | cut -d- -f1`</span>
-~$ <span class="userinput">echo "Site prefix is '$uuid_prefix'"</span>
-~$ <span class="userinput">read -rd $'\000' keepservice <<EOF; arv keep_service create --keep-service "$keepservice"</span>
-<span class="userinput">{
- "service_host":"<strong>keep0.$uuid_prefix.your.domain</strong>",
- "service_port":25107,
- "service_ssl_flag":false,
- "service_type":"disk"
-}
-EOF</span>
-</code></pre></notextile>
-
h3(#testing). Testing keep
Install the "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index b10a56e53..d13a93e60 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -26,7 +26,8 @@ Clusters:
SAMPLE:
InternalURLs:
"http://example.host:12345": {}
- SAMPLE: {}
+ SAMPLE:
+ Rendezvous: ""
ExternalURL: "-"
RailsAPI:
diff --git a/lib/config/deprecated_keepstore.go b/lib/config/deprecated_keepstore.go
index d9d3dc5ba..0d1a3c4e6 100644
--- a/lib/config/deprecated_keepstore.go
+++ b/lib/config/deprecated_keepstore.go
@@ -342,6 +342,9 @@ func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
} else {
volUUID = rendezvousUUID
}
+ si := cluster.Services.Keepstore.InternalURLs[myURL]
+ si.Rendezvous = uuid[12:]
+ cluster.Services.Keepstore.InternalURLs[myURL] = si
}
if volUUID == "" {
volUUID = newUUID(cluster.ClusterID, "nyw5e")
diff --git a/lib/config/deprecated_keepstore_test.go b/lib/config/deprecated_keepstore_test.go
index 42ea9bbc7..4891ceb8a 100644
--- a/lib/config/deprecated_keepstore_test.go
+++ b/lib/config/deprecated_keepstore_test.go
@@ -88,24 +88,35 @@ func (s *KeepstoreMigrationSuite) TestDeprecatedKeepstoreConfig(c *check.C) {
c.Assert(err, check.IsNil)
s.checkEquivalentWithKeepstoreConfig(c, `
-Listen: ":12345"
+Listen: ":25107"
Debug: true
LogFormat: text
MaxBuffers: 1234
MaxRequests: 2345
BlobSignatureTTL: 123m
BlobSigningKeyFile: `+keyfile.Name()+`
+Volumes:
+- Type: Directory
+ Root: /tmp
`, `
Clusters:
z1111:
- {}
+ SystemRootToken: `+arvadostest.AdminToken+`
+ TLS: {Insecure: true}
+ Services:
+ Controller:
+ ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
`, `
Clusters:
z1111:
+ SystemRootToken: `+arvadostest.AdminToken+`
+ TLS: {Insecure: true}
Services:
Keepstore:
InternalURLs:
- "http://`+hostname+`:12345": {}
+ "http://`+hostname+`:25107": {Rendezvous: `+s.ksByPort[25107].UUID[12:]+`}
+ Controller:
+ ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
SystemLogs:
Format: text
LogLevel: debug
@@ -115,6 +126,18 @@ Clusters:
Collections:
BlobSigningTTL: 123m
BlobSigningKey: blobsigningkey
+ Volumes:
+ z1111-nyw5e-`+s.ksByPort[25107].UUID[12:]+`:
+ AccessViaHosts:
+ "http://`+hostname+`:25107":
+ ReadOnly: false
+ Driver: Directory
+ DriverParameters:
+ Root: /tmp
+ Serialize: false
+ ReadOnly: false
+ Replication: 1
+ StorageClasses: {}
`)
}
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index c6be59ecf..89cfb8009 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -32,7 +32,8 @@ Clusters:
SAMPLE:
InternalURLs:
"http://example.host:12345": {}
- SAMPLE: {}
+ SAMPLE:
+ Rendezvous: ""
ExternalURL: "-"
RailsAPI:
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index c51f609f8..6f78073ac 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -294,7 +294,12 @@ func (su URL) String() string {
return (*url.URL)(&su).String()
}
-type ServiceInstance struct{}
+type ServiceInstance struct {
+ // Rendezvous is normally empty; when changing the URL of a
+ // Keepstore service, Rendezvous can be set to the old URL to
+ // preserve rendezvous ordering.
+ Rendezvous string `json:",omitempty"`
+}
type PostgreSQL struct {
Connection PostgreSQLConnection
diff --git a/services/api/app/controllers/arvados/v1/keep_services_controller.rb b/services/api/app/controllers/arvados/v1/keep_services_controller.rb
index 90b0a9228..865f9c942 100644
--- a/services/api/app/controllers/arvados/v1/keep_services_controller.rb
+++ b/services/api/app/controllers/arvados/v1/keep_services_controller.rb
@@ -10,17 +10,29 @@ class Arvados::V1::KeepServicesController < ApplicationController
def find_objects_for_index
# all users can list all keep services
- @objects = model_class.where('1=1')
+ @objects = from_config_or_db
super
end
def accessible
if request.headers['X-External-Client'] == '1'
- @objects = model_class.where('service_type=?', 'proxy')
+ @objects = from_config_or_db.where('service_type=?', 'proxy')
else
- @objects = model_class.where(model_class.arel_table[:service_type].not_eq('proxy'))
+ @objects = from_config_or_db.where('service_type<>?', 'proxy')
end
render_list
end
+ private
+
+ # return the set of keep services from the database (if this is an
+ # older installation or test system where entries have been added
+ # manually) or, preferably, the cluster config file.
+ def from_config_or_db
+ if KeepService.all.count == 0
+ KeepService.from_config
+ else
+ KeepService.all
+ end
+ end
end
diff --git a/services/api/app/models/keep_service.rb b/services/api/app/models/keep_service.rb
index bf5f3ccc0..f9dd01837 100644
--- a/services/api/app/models/keep_service.rb
+++ b/services/api/app/models/keep_service.rb
@@ -6,6 +6,9 @@ class KeepService < ArvadosModel
include HasUuid
include KindAndEtag
include CommonApiTemplate
+ extend DbCurrentTime
+
+ SERVER_START_TIME = db_current_time
api_accessible :user, extend: :common do |t|
t.add :service_host
@@ -26,4 +29,43 @@ class KeepService < ArvadosModel
def permission_to_update
current_user.andand.is_admin
end
+
+ def self.from_config
+ config_time = connection.quote(SERVER_START_TIME)
+ owner = connection.quote(system_user_uuid)
+ values = []
+ id = 1
+ Rails.configuration.Services.Keepstore.InternalURLs.each do |url, info|
+ values << "(#{id}, " + quoted_column_values_from_url(url: url.to_s, info: info).join(", ") + ", 'disk', 'f'::bool, #{config_time}, #{config_time}, #{owner}, #{owner}, null)"
+ id += 1
+ end
+ Rails.configuration.Services.Keepproxy.InternalURLs.each do |url, info|
+ values << "(#{id}, " + quoted_column_values_from_url(url: url.to_s, info: info).join(", ") + ", 'proxy', 'f'::bool, #{config_time}, #{config_time}, #{owner}, #{owner}, null)"
+ id += 1
+ end
+ if values.length == 0
+ # return empty set as AR relation
+ return where('1=0')
+ else
+ sql = "(values #{values.join(", ")}) as keep_services (id, uuid, service_host, service_port, service_ssl_flag, service_type, read_only, created_at, modified_at, owner_uuid, modified_by_user_uuid, modified_by_client_uuid)"
+ return KeepService.from(sql)
+ end
+ end
+
+ private
+
+ def self.quoted_column_values_from_url(url:, info:)
+ rvz = info.Rendezvous
+ rvz = url if rvz.blank?
+ if /^[a-zA-Z0-9]{15}$/ !~ rvz
+ # If rvz is an URL (either the real service URL, or an alternate
+ # one specified in config in order to preserve rendezvous order
+ # when changing hosts/ports), hash it to get 15 alphanums.
+ rvz = Digest::MD5.hexdigest(rvz)[0..15]
+ end
+ uuid = Rails.configuration.ClusterID + "-bi6l4-" + rvz
+ uri = URI::parse(url)
+ [uuid, uri.host, uri.port].map { |x| connection.quote(x) } + [(uri.scheme == 'https' ? "'t'::bool" : "'f'::bool")]
+ end
+
end
diff --git a/services/api/test/functional/arvados/v1/keep_services_controller_test.rb b/services/api/test/functional/arvados/v1/keep_services_controller_test.rb
index 41554be87..33c8aad3e 100644
--- a/services/api/test/functional/arvados/v1/keep_services_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/keep_services_controller_test.rb
@@ -36,4 +36,34 @@ class Arvados::V1::KeepServicesControllerTest < ActionController::TestCase
end
end
+ test "report configured servers if db is empty" do
+ KeepService.all.delete_all
+ expect_rvz = {}
+ n = 0
+ Rails.configuration.Services.Keepstore.InternalURLs.each do |k,v|
+ n += 1
+ rvz = "%015x" % n
+ expect_rvz[k.to_s] = rvz
+ Rails.configuration.Services.Keepstore.InternalURLs[k].Rendezvous = rvz
+ end
+ Rails.configuration.Services.Keepproxy.InternalURLs.each do |k,v|
+ n += 1
+ rvz = "%015x" % n
+ expect_rvz[k.to_s] = rvz
+ Rails.configuration.Services.Keepproxy.InternalURLs[k].Rendezvous = rvz
+ end
+ refute_empty expect_rvz
+ authorize_with :active
+ get :index,
+ params: {:format => :json},
+ headers: auth(:active)
+ json_response['items'].each do |svc|
+ url = "#{svc['service_ssl_flag'] ? 'https' : 'http'}://#{svc['service_host']}:#{svc['service_port']}"
+ assert_equal true, expect_rvz.has_key?(url), "#{url} does not match any configured service: expecting #{expect_rvz}"
+ assert_equal "zzzzz-bi6l4-#{expect_rvz[url]}", svc['uuid'], "exported service UUID should match InternalURLs.*.Rendezvous value"
+ expect_rvz.delete(url)
+ end
+ assert_equal({}, expect_rvz, "all configured Keepstore and Keepproxy services should be returned")
+ end
+
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list