[ARVADOS] updated: 1.3.0-1919-g2687f7570
Git user
git at public.curoverse.com
Mon Nov 25 20:32:53 UTC 2019
Summary of changes:
apps/workbench/Gemfile.lock | 4 +-
.../test/controllers/projects_controller_test.rb | 2 +-
build/package-build-dockerfiles/Makefile | 7 +-
.../{debian9 => debian10}/Dockerfile | 8 +-
build/package-test-dockerfiles/Makefile | 6 +-
.../{debian9 => debian10}/Dockerfile | 6 +-
build/package-testing/deb-common-test-packages.sh | 2 +-
...ckages-debian8.sh => test-packages-debian10.sh} | 0
build/run-build-packages-one-target.sh | 16 +-
build/run-build-packages-sso.sh | 4 +-
build/run-build-packages.sh | 21 +-
build/run-build-test-packages-one-target.sh | 4 +-
build/run-library.sh | 4 +-
build/run-tests.sh | 57 ++---
doc/Rakefile | 15 ++
doc/_config.yml | 11 +-
doc/_includes/_navbar_left.liquid | 4 +-
doc/admin/User account states.odg | Bin 0 -> 12087 bytes
doc/admin/activation.html.textile.liquid | 230 +--------------------
doc/admin/group-management.html.textile.liquid | 104 ++++++++++
....textile.liquid => logging.html.textile.liquid} | 8 +-
doc/admin/management-token.html.textile.liquid | 33 +--
doc/admin/migrating-providers.html.textile.liquid | 14 +-
doc/admin/reassign-ownership.html.textile.liquid | 51 +++++
doc/admin/troubleshooting.html.textile.liquid | 75 +------
.../user-management-cli.html.textile.liquid} | 37 ++--
doc/admin/user-management.html.textile.liquid | 185 +++++++++++++++++
doc/api/methods.html.textile.liquid | 5 +-
.../methods/container_requests.html.textile.liquid | 4 +-
doc/api/methods/containers.html.textile.liquid | 2 -
.../methods/user_agreements.html.textile.liquid | 44 ++++
doc/api/methods/users.html.textile.liquid | 32 ++-
doc/css/images.css | 5 +
doc/images/user-account-states.svg | 204 ++++++++++++++++++
doc/install/cheat_sheet.html.textile.liquid | 76 +------
lib/config/cmd.go | 27 ++-
lib/config/cmd_test.go | 18 +-
lib/config/config.default.yml | 8 +-
lib/config/export.go | 1 +
lib/config/generated_config.go | 8 +-
lib/controller/federation/conn.go | 9 +-
lib/controller/federation/generated.go | 39 +++-
lib/controller/federation/list.go | 20 +-
lib/controller/federation/list_test.go | 6 +-
lib/controller/federation/login_test.go | 4 +-
lib/controller/localdb/login.go | 3 +
lib/controller/localdb/login_test.go | 9 +-
lib/controller/router/router_test.go | 7 +
lib/controller/rpc/conn.go | 1 +
lib/dispatchcloud/container/queue.go | 2 +-
lib/dispatchcloud/scheduler/sync.go | 4 +-
.../tests/federation/arvbox-make-federation.cwl | 8 +
sdk/cwl/tests/federation/arvbox/fed-config.cwl | 6 +
sdk/cwl/tests/federation/arvbox/start.cwl | 23 ++-
sdk/go/arvados/collection.go | 8 +-
sdk/go/arvados/config.go | 1 +
sdk/go/arvados/container.go | 3 +
sdk/go/arvados/fs_collection.go | 6 +-
sdk/go/arvados/fs_deferred.go | 6 +-
sdk/go/arvados/specimen.go | 13 +-
sdk/python/arvados/commands/federation_migrate.py | 5 +-
sdk/python/tests/fed-migrate/CWLFile | 28 +++
.../tests/fed-migrate/arvbox-make-federation.cwl | 12 ++
sdk/python/tests/fed-migrate/check.py | 41 +++-
sdk/python/tests/fed-migrate/cwlex.cwl | 41 ++++
sdk/python/tests/fed-migrate/fed-migrate.cwl | 95 ++-------
sdk/python/tests/fed-migrate/fed-migrate.cwlex | 33 +--
sdk/python/tests/fed-migrate/set_login.py | 10 -
services/api/Gemfile.lock | 6 +-
services/api/app/models/arvados_model.rb | 3 +
services/api/app/models/user.rb | 2 +-
services/api/lib/record_filters.rb | 89 ++++----
.../v1/container_requests_controller_test.rb | 44 ++++
services/api/test/unit/permission_test.rb | 12 ++
services/keep-balance/collection.go | 8 +-
services/keep-balance/collection_test.go | 6 +-
services/login-sync/Gemfile.lock | 2 +-
tools/arvbox/lib/arvbox/docker/cluster-config.sh | 1 -
tools/arvbox/lib/arvbox/docker/go-setup.sh | 4 +-
.../lib/arvbox/docker/service/doc/run-service | 9 +-
80 files changed, 1272 insertions(+), 699 deletions(-)
copy build/package-build-dockerfiles/{debian9 => debian10}/Dockerfile (88%)
copy build/package-test-dockerfiles/{debian9 => debian10}/Dockerfile (85%)
copy build/package-testing/{test-packages-debian8.sh => test-packages-debian10.sh} (100%)
create mode 100644 doc/admin/User account states.odg
mode change 100644 => 120000 doc/admin/activation.html.textile.liquid
create mode 100644 doc/admin/group-management.html.textile.liquid
copy doc/admin/{troubleshooting.html.textile.liquid => logging.html.textile.liquid} (95%)
create mode 100644 doc/admin/reassign-ownership.html.textile.liquid
mode change 100644 => 120000 doc/admin/troubleshooting.html.textile.liquid
copy doc/{install/cheat_sheet.html.textile.liquid => admin/user-management-cli.html.textile.liquid} (55%)
create mode 100644 doc/admin/user-management.html.textile.liquid
create mode 100644 doc/api/methods/user_agreements.html.textile.liquid
create mode 100644 doc/images/user-account-states.svg
mode change 100644 => 120000 doc/install/cheat_sheet.html.textile.liquid
create mode 100644 sdk/python/tests/fed-migrate/CWLFile
create mode 100644 sdk/python/tests/fed-migrate/cwlex.cwl
delete mode 100644 sdk/python/tests/fed-migrate/set_login.py
via 2687f75703b776e2f2d2840ebf4d8d0e88ea68cb (commit)
via 5ee93e408c0e547dfb03b2f3d039a7715126395b (commit)
via 607d2b1190bbba7c018fcbc91e66fdb0db1ce90f (commit)
via 5ffd4a118ae8c97c4c75fc789b2b5c8437461797 (commit)
via 2e6208ee675505d9af3a9d7c837aabdf2d497f21 (commit)
via e087a608834328786d0aec7956646fdf63b5ed44 (commit)
via 0e2a467a923bd490f3e1dc4d8c00a9e2f29e59d4 (commit)
via 7737b743baed690e98882a1bb3e44d1a76e45030 (commit)
via 6361b15d39bdb78b5b1f8ca696fae543c677848c (commit)
via 66f6a487050e1cdb3fda6a32f195eb1913c83241 (commit)
via 1b6b00b1f3f1a1931c27c69f1624d2c34e94a880 (commit)
via ed5be18ec61d2edecbf878785633aea2b056b20a (commit)
via f3de4ad1c15b90edea0d166b4e48630bb191ac76 (commit)
via 06c9ea02435fb56e8c32a31694a414ca161dc2ce (commit)
via 76509a7932f48f945904163fd414b60da1d601a0 (commit)
via 086c972bae004544cb43d9ea15388d486d3cd1ed (commit)
via 35ed9c4e39306ddbeebdf0445149be72c1cc5284 (commit)
via 0c7319e50b1e5c35d8645b26758f19bce0f32cb0 (commit)
via c73f87c5fedba20b82a091152353f718d81ba6c3 (commit)
via 92d7d779d4f0415d09f536d286972bd953e7b566 (commit)
via e4b1a161bdc5a95b9dfd53a3b74bfa552bbc0f18 (commit)
via 002aec2a7db39d269bb3c9123783022c2e32a5cc (commit)
via 4ecc670983133293c49bacfb4815a4b18cf2cdee (commit)
via ae30520a25d494e7674fe53138f5cefad9c199a6 (commit)
via 3994322d0ae878ac0055d542157f01b09e3a2860 (commit)
via 6b1037d37dfeca503289cb54304189a6774db4e9 (commit)
via a729ffba72b79a8fbe7ba88c0a1c40c6ac5c48bf (commit)
via 5ad819e0aa69a8ea453abe18a8e00aa93daa90a2 (commit)
via 40e9a645da140a6b94497d1a79ba8783d22421f7 (commit)
via a1f06e4e118923d301683824ccd47bcf68c8ef69 (commit)
via 9d53a848044dab5daaeb76c65833afbd4748a33e (commit)
via bdc8a7630030494c63fb0426be4c15a93a9a37cb (commit)
via 542a72e8ea402a65d75a5251ba219341834fb2c9 (commit)
via ee341f1fc2579fdf28a8b8f07918f9ae195b0727 (commit)
via 98e4a92f007533b2924604e4f83da9a6d15e0ef3 (commit)
via 8b43f32b2c11d45f951bf4ff1bffab03d391ff41 (commit)
via ea42eaf1e8776704de1ef75d01259d33dcabaef3 (commit)
via 943827578884b09a155443a9d2bb685a327070f9 (commit)
via 82d733300a8dd1859b9cd06f3989a2fe30c43887 (commit)
via f18f0e90022f7a78032d5e4ebf9995d5256a076e (commit)
via 89f10cd956b1282d8d650fd82f945f8169664df9 (commit)
via d30db61ea4bdcd686c418cef01ba00e89c4bc338 (commit)
via e6253e06dd350acb6f04b32785baa29bf269acfd (commit)
from 233a2b6bd23a3e2054cfc0690f2bc06c0f9f7323 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2687f75703b776e2f2d2840ebf4d8d0e88ea68cb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Nov 25 15:32:21 2019 -0500
15720: Warn about empty ManagementToken or SystemRootToken.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/doc/admin/management-token.html.textile.liquid b/doc/admin/management-token.html.textile.liquid
index 5380f38f9..cf3e273ce 100644
--- a/doc/admin/management-token.html.textile.liquid
+++ b/doc/admin/management-token.html.textile.liquid
@@ -16,17 +16,6 @@ Services must have ManagementToken configured. This is used to authorize access
To access a monitoring endpoint, the requester must provide the HTTP header @Authorization: Bearer (ManagementToken)@.
-h2. API server
-
-Set @ManagementToken@ in the appropriate section of @application.yml@
-
-<pre>
-production:
- # Token to be included in all healthcheck requests. Disabled by default.
- # Server expects request header of the format "Authorization: Bearer xxx"
- ManagementToken: xxx
-</pre>
-
h2. Node Manager
Set @port@ (the listen port) and @ManagementToken@ in the @Manage@ section of @node-manager.ini at .
@@ -45,12 +34,26 @@ Set @port@ (the listen port) and @ManagementToken@ in the @Manage@ section of @n
ManagementToken = xxx
</pre>
-h2. Other services
+h2. API server and other services
-The following services also support monitoring. Set @ManagementToken@ in the respective yaml config file for each service.
+The following services also support monitoring.
+* API server
+* arv-git-httpd
+* controller
+* keep-balance
+* keepproxy
* keepstore
* keep-web
-* keepproxy
-* arv-git-httpd
* websockets
+
+Set @ManagementToken@ in the appropriate section of @/etc/arvados/config.yml at .
+
+<notextile>
+<pre><code>Clusters:
+ <span class="userinput">uuid_prefix</span>:
+ # Token to be included in all healthcheck requests. Disabled by default.
+ # Server expects request header of the format "Authorization: Bearer xxx"
+ ManagementToken: xxx
+</code></pre>
+</notextile>
diff --git a/lib/config/cmd.go b/lib/config/cmd.go
index e9ceaca86..1ca278391 100644
--- a/lib/config/cmd.go
+++ b/lib/config/cmd.go
@@ -12,6 +12,7 @@ import (
"os"
"os/exec"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"github.com/ghodss/yaml"
"github.com/sirupsen/logrus"
@@ -124,6 +125,10 @@ func (checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo
if err != nil {
return 1
}
+ problems := false
+ if warnAboutProblems(logger, withDepr) {
+ problems = true
+ }
cmd := exec.Command("diff", "-u", "--label", "without-deprecated-configs", "--label", "relying-on-deprecated-configs", "/dev/fd/3", "/dev/fd/4")
for _, obj := range []interface{}{withoutDepr, withDepr} {
y, _ := yaml.Marshal(obj)
@@ -153,7 +158,27 @@ func (checkCommand) RunCommand(prog string, args []string, stdin io.Reader, stdo
if logbuf.Len() > 0 {
return 1
}
- return 0
+
+ if problems {
+ return 1
+ } else {
+ return 0
+ }
+}
+
+func warnAboutProblems(logger logrus.FieldLogger, cfg *arvados.Config) bool {
+ warned := false
+ for id, cc := range cfg.Clusters {
+ if cc.SystemRootToken == "" {
+ logger.Warnf("Clusters.%s.SystemRootToken is empty; see https://doc.arvados.org/master/install/install-keepstore.html", id)
+ warned = true
+ }
+ if cc.ManagementToken == "" {
+ logger.Warnf("Clusters.%s.ManagementToken is empty; see https://doc.arvados.org/admin/management-token.html", id)
+ warned = true
+ }
+ }
+ return warned
}
var DumpDefaultsCommand defaultsCommand
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index fb1cba38b..c275e4c35 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -30,25 +30,27 @@ func (s *CommandSuite) SetUpSuite(c *check.C) {
os.Unsetenv("ARVADOS_API_TOKEN")
}
-func (s *CommandSuite) TestBadArg(c *check.C) {
+func (s *CommandSuite) TestDump_BadArg(c *check.C) {
var stderr bytes.Buffer
code := DumpCommand.RunCommand("arvados config-dump", []string{"-badarg"}, bytes.NewBuffer(nil), bytes.NewBuffer(nil), &stderr)
c.Check(code, check.Equals, 2)
c.Check(stderr.String(), check.Matches, `(?ms)flag provided but not defined: -badarg\nUsage:\n.*`)
}
-func (s *CommandSuite) TestEmptyInput(c *check.C) {
+func (s *CommandSuite) TestDump_EmptyInput(c *check.C) {
var stdout, stderr bytes.Buffer
code := DumpCommand.RunCommand("arvados config-dump", []string{"-config", "-"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(code, check.Equals, 1)
c.Check(stderr.String(), check.Matches, `config does not define any clusters\n`)
}
-func (s *CommandSuite) TestCheckNoDeprecatedKeys(c *check.C) {
+func (s *CommandSuite) TestCheck_NoWarnings(c *check.C) {
var stdout, stderr bytes.Buffer
in := `
Clusters:
z1234:
+ ManagementToken: xyzzy
+ SystemRootToken: xyzzy
API:
MaxItemsPerResponse: 1234
PostgreSQL:
@@ -73,7 +75,7 @@ Clusters:
c.Check(stderr.String(), check.Equals, "")
}
-func (s *CommandSuite) TestCheckDeprecatedKeys(c *check.C) {
+func (s *CommandSuite) TestCheck_DeprecatedKeys(c *check.C) {
var stdout, stderr bytes.Buffer
in := `
Clusters:
@@ -86,7 +88,7 @@ Clusters:
c.Check(stdout.String(), check.Matches, `(?ms).*\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
}
-func (s *CommandSuite) TestCheckOldKeepstoreConfigFile(c *check.C) {
+func (s *CommandSuite) TestCheck_OldKeepstoreConfigFile(c *check.C) {
f, err := ioutil.TempFile("", "")
c.Assert(err, check.IsNil)
defer os.Remove(f.Name())
@@ -106,7 +108,7 @@ Clusters:
c.Check(stderr.String(), check.Matches, `(?ms).*you should remove the legacy keepstore config file.*\n`)
}
-func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
+func (s *CommandSuite) TestCheck_UnknownKey(c *check.C) {
var stdout, stderr bytes.Buffer
in := `
Clusters:
@@ -130,7 +132,7 @@ Clusters:
c.Check(stderr.String(), check.Matches, `(?ms).*unexpected object in config entry: Clusters.z1234.PostgreSQL.ConnectionPool"\n.*`)
}
-func (s *CommandSuite) TestDumpFormatting(c *check.C) {
+func (s *CommandSuite) TestDump_Formatting(c *check.C) {
var stdout, stderr bytes.Buffer
in := `
Clusters:
@@ -149,7 +151,7 @@ Clusters:
c.Check(stdout.String(), check.Matches, `(?ms).*http://localhost:12345: {}\n.*`)
}
-func (s *CommandSuite) TestDumpUnknownKey(c *check.C) {
+func (s *CommandSuite) TestDump_UnknownKey(c *check.C) {
var stdout, stderr bytes.Buffer
in := `
Clusters:
commit 5ee93e408c0e547dfb03b2f3d039a7715126395b
Merge: 233a2b6bd 607d2b119
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Nov 25 15:20:21 2019 -0500
15720: Merge branch 'master' into 15720-fed-user-list
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --cc lib/controller/federation/generated.go
index 961cd5a40,fb91a8496..0a6664498
--- a/lib/controller/federation/generated.go
+++ b/lib/controller/federation/generated.go
@@@ -16,9 -17,11 +17,11 @@@ import
// -- this file is auto-generated -- do not edit -- edit list.go and run "go generate" instead --
//
-func (conn *Conn) ContainerList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerList, error) {
+func (conn *Conn) generated_ContainerList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerList, error) {
var mtx sync.Mutex
var merged arvados.ContainerList
+ var needSort atomic.Value
+ needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
cl, err := backend.ContainerList(ctx, options)
if err != nil {
@@@ -41,9 -57,11 +57,11 @@@
return merged, err
}
-func (conn *Conn) SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error) {
+func (conn *Conn) generated_SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error) {
var mtx sync.Mutex
var merged arvados.SpecimenList
+ var needSort atomic.Value
+ needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
cl, err := backend.SpecimenList(ctx, options)
if err != nil {
@@@ -62,31 -81,18 +81,43 @@@
}
return uuids, nil
})
- sort.Slice(merged.Items, func(i, j int) bool { return merged.Items[i].UUID < merged.Items[j].UUID })
+ if needSort.Load().(bool) {
+ // Apply the default/implied order, "modified_at desc"
+ sort.Slice(merged.Items, func(i, j int) bool {
+ mi, mj := merged.Items[i].ModifiedAt, merged.Items[j].ModifiedAt
+ return mj.Before(mi)
+ })
+ }
+ if merged.Items == nil {
+ // Return empty results as [], not null
+ // (https://github.com/golang/go/issues/27589 might be
+ // a better solution in the future)
+ merged.Items = []arvados.Specimen{}
+ }
return merged, err
}
+
+func (conn *Conn) generated_UserList(ctx context.Context, options arvados.ListOptions) (arvados.UserList, error) {
+ var mtx sync.Mutex
+ var merged arvados.UserList
+ err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
+ cl, err := backend.UserList(ctx, options)
+ if err != nil {
+ return nil, err
+ }
+ mtx.Lock()
+ defer mtx.Unlock()
+ if len(merged.Items) == 0 {
+ merged = cl
+ } else {
+ merged.Items = append(merged.Items, cl.Items...)
+ }
+ uuids := make([]string, 0, len(cl.Items))
+ for _, item := range cl.Items {
+ uuids = append(uuids, item.UUID)
+ }
+ return uuids, nil
+ })
+ sort.Slice(merged.Items, func(i, j int) bool { return merged.Items[i].UUID < merged.Items[j].UUID })
+ return merged, err
+}
diff --cc lib/controller/federation/list.go
index 7178d7b0a,54f59812a..26b6b254e
--- a/lib/controller/federation/list.go
+++ b/lib/controller/federation/list.go
@@@ -20,9 -21,11 +21,11 @@@ import
// CollectionList is used as a template to auto-generate List()
// methods for other types; see generate.go.
-func (conn *Conn) CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error) {
+func (conn *Conn) generated_CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error) {
var mtx sync.Mutex
var merged arvados.CollectionList
+ var needSort atomic.Value
+ needSort.Store(false)
err := conn.splitListRequest(ctx, options, func(ctx context.Context, _ string, backend arvados.API, options arvados.ListOptions) ([]string, error) {
cl, err := backend.CollectionList(ctx, options)
if err != nil {
diff --cc lib/controller/federation/list_test.go
index 5a630a945,35d201028..a9c4f588f
--- a/lib/controller/federation/list_test.go
+++ b/lib/controller/federation/list_test.go
@@@ -8,9 -8,18 +8,10 @@@ import
"context"
"fmt"
"net/http"
- "net/url"
- "os"
+ "sort"
- "testing"
- "git.curoverse.com/arvados.git/lib/controller/router"
- "git.curoverse.com/arvados.git/lib/controller/rpc"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
- "git.curoverse.com/arvados.git/sdk/go/auth"
- "git.curoverse.com/arvados.git/sdk/go/ctxlog"
- "git.curoverse.com/arvados.git/sdk/go/httpserver"
check "gopkg.in/check.v1"
)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list