[ARVADOS] updated: 2.1.0-213-g82469cf3c
Git user
git at public.arvados.org
Thu Dec 10 20:35:17 UTC 2020
Summary of changes:
.gitignore | 2 +
.../app/controllers/application_controller.rb | 2 +-
apps/workbench/app/controllers/users_controller.rb | 12 +
apps/workbench/app/views/users/profile.html.erb | 21 +-
build/build-dev-docker-jobs-image.sh | 4 +-
build/run-build-docker-jobs-image.sh | 45 +-
build/run-build-packages-one-target.sh | 3 +-
build/run-build-packages-python-and-ruby.sh | 13 +-
build/run-build-packages.sh | 3 +
build/run-library.sh | 30 +-
cmd/arvados-client/cmd.go | 2 +
doc/_config.yml | 1 +
doc/admin/config.html.textile.liquid | 2 +-
doc/admin/federation.html.textile.liquid | 4 +-
doc/admin/upgrading.html.textile.liquid | 2 +-
doc/admin/user-activity.html.textile.liquid | 101 ++++
doc/api/keep-s3.html.textile.liquid | 21 +-
doc/api/methods.html.textile.liquid | 6 +-
doc/api/methods/jobs.html.textile.liquid | 16 +-
.../methods/pipeline_templates.html.textile.liquid | 16 +-
.../arvados-on-kubernetes-GKE.html.textile.liquid | 8 -
...ados-on-kubernetes-minikube.html.textile.liquid | 6 +-
.../arvados-on-kubernetes.html.textile.liquid | 4 -
doc/install/arvbox.html.textile.liquid | 19 +-
doc/install/install-api-server.html.textile.liquid | 18 +-
doc/install/install-keep-web.html.textile.liquid | 4 +-
doc/install/salt-single-host.html.textile.liquid | 135 ++++-
doc/install/salt-vagrant.html.textile.liquid | 58 ++-
doc/sdk/python/cookbook.html.textile.liquid | 31 ++
.../tutorials/wgs-tutorial.html.textile.liquid | 4 +-
lib/boot/postgresql.go | 4 +-
lib/config/config.default.yml | 2 +
lib/config/generated_config.go | 2 +
lib/controller/fed_containers.go | 6 +-
lib/controller/federation.go | 8 +
lib/controller/federation/conn.go | 15 +-
lib/controller/handler.go | 1 +
lib/controller/integration_test.go | 77 +++
lib/controller/localdb/conn.go | 12 +
lib/controller/router/response.go | 42 ++
lib/controller/rpc/conn.go | 2 -
lib/controller/rpc/conn_test.go | 6 +-
.../command.go => costanalyzer/cmd.go} | 8 +-
lib/costanalyzer/costanalyzer.go | 568 +++++++++++++++++++++
lib/costanalyzer/costanalyzer_test.go | 325 ++++++++++++
lib/crunchrun/background.go | 2 +-
lib/crunchrun/crunchrun.go | 34 +-
lib/crunchrun/crunchrun_test.go | 8 +-
lib/ctrlctx/db.go | 1 +
lib/install/deps.go | 4 +-
lib/mount/command.go | 1 +
sdk/cwl/arvados_cwl/__init__.py | 2 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 7 +-
sdk/cwl/arvados_cwl/arvdocker.py | 6 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 2 +-
sdk/cwl/arvados_cwl/executor.py | 2 +-
sdk/cwl/arvados_cwl/pathmapper.py | 4 +-
sdk/cwl/arvados_cwl/runner.py | 21 +-
sdk/cwl/arvados_cwl/task_queue.py | 77 ---
sdk/cwl/arvados_version.py | 34 +-
sdk/cwl/gittaggers.py | 48 --
sdk/cwl/setup.py | 2 +-
sdk/cwl/test_with_arvbox.sh | 4 +-
sdk/cwl/tests/test_submit.py | 36 +-
sdk/cwl/tests/test_tq.py | 2 +-
sdk/go/arvados/container.go | 2 +-
sdk/go/arvadostest/db.go | 1 +
sdk/go/arvadostest/fixtures.go | 25 +-
sdk/go/blockdigest/blockdigest_test.go | 4 +-
sdk/go/keepclient/hashcheck.go | 32 +-
sdk/go/keepclient/keepclient.go | 2 +-
sdk/go/keepclient/keepclient_test.go | 12 +-
sdk/go/keepclient/support.go | 34 +-
sdk/python/arvados/util.py | 61 +++
sdk/python/arvados_version.py | 30 +-
sdk/python/gittaggers.py | 29 --
sdk/python/tests/run_test_server.py | 15 +-
sdk/python/tests/test_util.py | 137 +++++
.../api/app/models/api_client_authorization.rb | 32 +-
services/api/app/models/collection.rb | 7 +-
...02174753_fix_collection_versions_timestamps.rb} | 8 +-
services/api/db/structure.sql | 3 +-
services/api/lib/create_superuser_token.rb | 2 +-
.../api/lib/fix_collection_versions_timestamps.rb | 43 ++
services/api/test/fixtures/collections.yml | 102 +++-
services/api/test/fixtures/container_requests.yml | 234 ++++++++-
services/api/test/fixtures/containers.yml | 147 ++++++
services/api/test/unit/collection_test.rb | 29 +-
.../api/test/unit/create_superuser_token_test.rb | 27 +-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 2 +-
services/dockercleaner/arvados_version.py | 26 +-
services/dockercleaner/gittaggers.py | 1 -
services/fuse/arvados_version.py | 30 +-
services/fuse/gittaggers.py | 1 -
services/keep-web/handler.go | 59 ++-
services/keep-web/handler_test.go | 35 +-
services/keep-web/s3.go | 126 +++--
services/keep-web/s3_test.go | 105 +++-
services/keep-web/server_test.go | 18 +-
tools/arvbox/bin/arvbox | 1 +
tools/crunchstat-summary/arvados_version.py | 30 +-
tools/crunchstat-summary/gittaggers.py | 1 -
tools/salt-install/README.md | 2 +-
tools/salt-install/Vagrantfile | 13 +-
tools/salt-install/provision.sh | 120 +++--
tools/salt-install/single_host/arvados.sls | 32 +-
.../salt-install/single_host/docker.sls | 7 +-
.../single_host/nginx_api_configuration.sls | 2 +-
.../single_host/nginx_controller_configuration.sls | 5 +-
.../single_host/nginx_keepproxy_configuration.sls | 5 +-
.../single_host/nginx_keepweb_configuration.sls | 5 +-
.../single_host/nginx_webshell_configuration.sls | 5 +-
.../single_host/nginx_websocket_configuration.sls | 5 +-
.../single_host/nginx_workbench2_configuration.sls | 5 +-
.../single_host/nginx_workbench_configuration.sls | 7 +-
tools/salt-install/tests/hasher-workflow-job.yml | 10 +
tools/salt-install/tests/hasher-workflow.cwl | 65 +++
.../salt-install/tests/hasher.cwl | 25 +-
tools/salt-install/tests/run-test.sh | 68 +++
.../salt-install/tests/test.txt | 2 +-
{services/fuse => tools/user-activity}/MANIFEST.in | 1 -
tools/user-activity/README.rst | 5 +
agpl-3.0.txt => tools/user-activity/agpl-3.0.txt | 0
.../arvados_user_activity/__init__.py | 2 -
tools/user-activity/arvados_user_activity/main.py | 152 ++++++
.../user-activity}/arvados_version.py | 30 +-
.../user-activity/bin/arv-user-activity | 5 +-
.../fpm-info.sh | 0
.../{crunchstat-summary => user-activity}/setup.py | 25 +-
129 files changed, 3325 insertions(+), 699 deletions(-)
create mode 100644 doc/admin/user-activity.html.textile.liquid
copy lib/{deduplicationreport/command.go => costanalyzer/cmd.go} (77%)
create mode 100644 lib/costanalyzer/costanalyzer.go
create mode 100644 lib/costanalyzer/costanalyzer_test.go
delete mode 100644 sdk/cwl/arvados_cwl/task_queue.py
delete mode 100644 sdk/cwl/gittaggers.py
delete mode 100644 sdk/python/gittaggers.py
copy services/api/db/migrate/{20200602141328_fix_roles_projects.rb => 20201202174753_fix_collection_versions_timestamps.rb} (53%)
create mode 100644 services/api/lib/fix_collection_versions_timestamps.rb
delete mode 120000 services/dockercleaner/gittaggers.py
delete mode 120000 services/fuse/gittaggers.py
delete mode 120000 tools/crunchstat-summary/gittaggers.py
copy apps/workbench/app/controllers/keep_services_controller.rb => tools/salt-install/single_host/docker.sls (62%)
create mode 100644 tools/salt-install/tests/hasher-workflow-job.yml
create mode 100644 tools/salt-install/tests/hasher-workflow.cwl
copy sdk/cwl/tests/stdout.cwl => tools/salt-install/tests/hasher.cwl (50%)
create mode 100755 tools/salt-install/tests/run-test.sh
copy sdk/cwl/tests/__init__.py => tools/salt-install/tests/test.txt (95%)
copy {services/fuse => tools/user-activity}/MANIFEST.in (72%)
create mode 100644 tools/user-activity/README.rst
copy agpl-3.0.txt => tools/user-activity/agpl-3.0.txt (100%)
copy services/api/config/initializers/andand.rb => tools/user-activity/arvados_user_activity/__init__.py (84%)
create mode 100755 tools/user-activity/arvados_user_activity/main.py
copy {services/fuse => tools/user-activity}/arvados_version.py (60%)
copy services/dockercleaner/tests/__init__.py => tools/user-activity/bin/arv-user-activity (63%)
mode change 100644 => 100755
copy tools/{crunchstat-summary => user-activity}/fpm-info.sh (100%)
copy tools/{crunchstat-summary => user-activity}/setup.py (52%)
via 82469cf3c2075f738acef86ee8cb9d39f49e5589 (commit)
via f437716f309592d9b2ce801c344b82c1ddcd7e39 (commit)
via 82313a238872ed2143679ee18b4a49eef7bd39c1 (commit)
via ff37086ed63714d4eb8f35b34de761b47fe15cde (commit)
via 44f0fda05ab9b3cc93205a4365edc0a699fa7957 (commit)
via 0f7fab52b7ca43763bd17fd414009319bd653e51 (commit)
via 90a0d1808ec312cd3bd5ebe44edbe2b5fed999db (commit)
via 150f7d25a49f33fa8d2f2b22ea002f573eae2cb2 (commit)
via a8f0e996f2a38995cd97f344b76a692401df10b2 (commit)
via 65f0da0b0c8c999274422763d1825f19ccda660e (commit)
via fe352f6d7d9b9aa338295f5f3abde0da07131f93 (commit)
via d5efe792b79cd736e68efab107887ae28b3cf0b6 (commit)
via c18025fcc5c2c782a35e7c9c582489be623e671a (commit)
via cdee3de0f2cd7d8169fbc6cc052c0db4fcfc9369 (commit)
via 7919e888b7028b803ae7ab9cd4adfae3fe413d63 (commit)
via d0301d9ddbf002fc227bbb56274e7e0446187a6f (commit)
via 08dcbff16dca8ebfd5665afc8850c04ca6fee51e (commit)
via 799bde4ed30dbd73fbbd0b59dc6d6d9f900dea8a (commit)
via 7e4e9c15e6416115f53e511eb121c4d667bca61e (commit)
via 7f84cbbc200a15c08a35c341af413f6944b4ce6d (commit)
via f3bc4d4bf4771cb2b4ea5cf2db822e4c21f5820b (commit)
via 7b0ef99243c37816540a09670f7782c681cfb760 (commit)
via 8dfbae733b7fa9e9ee579c1fbc1832652cc17485 (commit)
via e3c1d52294b815eff176ce8e62ddb0153888bee0 (commit)
via 5ea6e759865193c415f86018a07a012731e86a2b (commit)
via e187e11f44da338014088312c832bfabc3856f0e (commit)
via 33c97ad53a33e005471254cf0b08372c4dabeb4d (commit)
via 57ef73c24043dcf5a315ba92eff3a849797b19d0 (commit)
via 070fcdf489618c3318ada347a876e2e6d21dd0e4 (commit)
via 320375e3c0071fb3f492323bc3c18e4ca7605fff (commit)
via 2fd2dab35b25b09d4923c741bb9fc364299a1a6d (commit)
via 152c62fd0cde25378ee0d2d10c86946dcec8c138 (commit)
via b16a207b45d04345a905fe0ac9336eea011026a6 (commit)
via 6ff4379143fe5409eea488d583921b3a68ecb3a2 (commit)
via 17832911c76d4d7f22468b764152fd53f48f0aa2 (commit)
via 599507428ab48bfea5dd9a1f25c9ad1f84ab2b0c (commit)
via 486e297e57ab63c5dda916ae24f8e9c29ce30b66 (commit)
via b379f3bdfa8806947c64e00d81f01e35874c8d45 (commit)
via c924a8fb59cf1455a9ba64726530236d6bc8f141 (commit)
via a028f9707926070d8309ceda2b9f88f8625a6ea1 (commit)
via f563387fb894b0f459d40d4550cf731d930e2bf6 (commit)
via 77a773281a967477799d4a586cec59ba40a66ae2 (commit)
via 70f4b1a134dd64c18498c880d218bfc8187349ed (commit)
via 06227b8b8a1d5c3488a35342444dbb2aa59684e5 (commit)
via 41cdc93d9aa3b119322bf1e666c96d235c36e43c (commit)
via 20b80f0f28588b3c7961f5ecdfc8af2659affabc (commit)
via bf53079ba22cfa04b3fbac5b8dcf4467844601f8 (commit)
via dc5611c404d0d32994b07037309935ec717c12c2 (commit)
via 1abc152750e2780c8c388f28825ba7594ee55722 (commit)
via 3b9c8adfe9f4cf8966d70fad927affaae0a11802 (commit)
via ffc10a3d7cede7651eb1b78353be907cff8758b4 (commit)
via cf2d4da8f988451d7a3f1d37f5a12e4fcfc2f0c4 (commit)
via 3b6a64460d926e198a362293c6dc2b349b142c2d (commit)
via e40626646448198d33ae7f8f1024fc91c6d1649a (commit)
via 33ef9263f9e574768d95fbc65ac302093a7158ad (commit)
via e916ddcdf18b8c9aad35c2a810c4636ad805e7bf (commit)
via e8ad88b12724d5102b5e0dde85014fd86c6827ef (commit)
via 7e7aaec01af5c9b2547478529812a66ea4af6bbf (commit)
via 16f07140d31504341faf1158c4b8eccf1771bd7a (commit)
via 106e69e3bc04d52ac65dfc19463215817a75320e (commit)
via 5513ae5fa4abed9b3f871173f08d4a0aa52418af (commit)
via c9e619571b8be5d6576c9a3f74e877bf39ce02c9 (commit)
via 51251e7473df5f1a9036d36f2d38d9dd1788f9cc (commit)
via 77adea164640b48948ccd8292f11d9354d2ebd6b (commit)
via 1944cf47cac3607ab69b31da4adde434662f62c5 (commit)
via 6452fa1e968d0dd33eb2628fec5323bf4f8bbe8c (commit)
via deaadbea28f273c4528394606a18a9443b30ea02 (commit)
via b6122315820c594ac4d1d5b157dd90523a79946a (commit)
via 2325be1cf9fc9cd34ae826688347cf7a9c01e9aa (commit)
via b377684bf4c7a6211e39556c744544857ee66493 (commit)
via a31656998dc52d163c6be7c40918daa1d5f16048 (commit)
via f7499eb0eeb8f9cbfad1545f7e196f8939fb0f05 (commit)
via d5d15b19d43ba894912d710e9baa1538d5f29ef3 (commit)
via d577b0e4f583d5aa4cb01af81ee0b326d9f81f29 (commit)
via 7995f95d0989b3974c9f1d39da4f36c74ae958cf (commit)
via a15943fc33acd69fb4b706fb97c10635a4a9dbb5 (commit)
via 568168423fa929cbfa05d3c6bca591017ba00e44 (commit)
via 076d6df6431377b1af8ca94491e3ef7c1f21052a (commit)
via 4ecea81f99d1ba947e1b0bb1eaed22e6d88dabe5 (commit)
via e6c3e65fb6c5d80ba4364665b96cc3d9a7f3ac00 (commit)
via ff711779ea897ba208e3c6880e53fa98db484e51 (commit)
via 456fc0518fba438448c34d233a8c68371aa6bffe (commit)
via 047dab2b789c6db8cb43624dd284f703f8903268 (commit)
via 9e0399b7fcff3ea83ee986b07f60b8b27659d5c9 (commit)
via 8f2b8255ab3e58911b56942530e322e0fcd954e2 (commit)
via fff3ea11c8658d1363e5d2da4a4a7df3055b8f76 (commit)
via 3dc9a7d7830c84f5c235c706f473066589a5c6cd (commit)
via 8ffbca22ca9f6172fe52a5ae2de40a6b7d537978 (commit)
via 48a6fe360bd9f5ad59c7e5114898b98a2413e898 (commit)
via a06f784c9f8d2acb2ec9baa852193117b8c1cfdd (commit)
via 3e1c102325332298de9a2d0de74026e846a0d9af (commit)
via f8c45afb252a2b6620a14594c874bca3217fb6e3 (commit)
via 04df85b3ace293a3c1ad59e7ad279f53dbdf14fa (commit)
via f6c012b2e08779385aaf98c15069febb8d873820 (commit)
via 4bd8ba40bab2ddd229f921d2fa709e0c19edf371 (commit)
via 771e7c9387650565dbc87656d78e987ae43ba91b (commit)
from 91d455dae37fae429421dae2c02e2cfce3d51799 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 82469cf3c2075f738acef86ee8cb9d39f49e5589
Author: Nico Cesar <nico at nicocesar.com>
Date: Tue Nov 10 09:00:52 2020 -0500
container requests in the new codepath for controller
Controller's "new" codepath is a step towards sunsetting railsapi. This
commit is the initial work to get Container Requests in that path
The reason for deleting some tests is intentional. This is part of the migration
from
apps/workbench/test/controllers/container_requests_controller_test.rb
to
lib/controller/integration_test.go
This single-commit is the result of collaboration with Lucas Di Pentima
and Tom Clegg in the branch 17014-controller-cr
Arvados-DCO-1.1-Signed-off-by: Nico Cesar <nico at curii.com>
diff --git a/lib/controller/federation.go b/lib/controller/federation.go
index bc93cc9ea..63f21727b 100644
--- a/lib/controller/federation.go
+++ b/lib/controller/federation.go
@@ -165,6 +165,14 @@ func (h *Handler) validateAPItoken(req *http.Request, token string) (*CurrentUse
token = sp[2]
}
+<<<<<<< HEAD
+=======
+ if len(token) < 41 {
+ ctxlog.FromContext(req.Context()).Debugf("validateAPItoken(%s): The lenght of the token is not 41", token)
+ return nil, false, nil
+ }
+
+>>>>>>> 611e6e5fd (container requests in the new codepath for controller)
user.Authorization.APIToken = token
var scopes string
err = db.QueryRowContext(req.Context(), `SELECT api_client_authorizations.uuid, api_client_authorizations.scopes, users.uuid FROM api_client_authorizations JOIN users on api_client_authorizations.user_id=users.id WHERE api_token=$1 AND (expires_at IS NULL OR expires_at > current_timestamp AT TIME ZONE 'UTC') LIMIT 1`, token).Scan(&user.Authorization.UUID, &scopes, &user.UUID)
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index 247556e33..ffca3b117 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -10,7 +10,6 @@ import (
"encoding/json"
"errors"
"fmt"
- "log"
"net/http"
"net/url"
"regexp"
@@ -343,7 +342,6 @@ func (conn *Conn) ContainerRequestList(ctx context.Context, options arvados.List
func (conn *Conn) ContainerRequestCreate(ctx context.Context, options arvados.CreateOptions) (arvados.ContainerRequest, error) {
be := conn.chooseBackend(options.ClusterID)
- log.Printf("THIS IS federation.Conn.ContainerRequestCreate() for %s we are %s", options.ClusterID, conn.cluster.ClusterID)
if be == conn.local {
return be.ContainerRequestCreate(ctx, options)
}
diff --git a/lib/controller/localdb/conn.go b/lib/controller/localdb/conn.go
index d197675f8..80633781c 100644
--- a/lib/controller/localdb/conn.go
+++ b/lib/controller/localdb/conn.go
@@ -31,17 +31,29 @@ func NewConn(cluster *arvados.Cluster) *Conn {
return &conn
}
+<<<<<<< HEAD
// Logout handles the logout of conn giving to the appropriate loginController
+=======
+// Logout handles the logout of conn giving to the appropiate loginController
+>>>>>>> 611e6e5fd (container requests in the new codepath for controller)
func (conn *Conn) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
return conn.loginController.Logout(ctx, opts)
}
+<<<<<<< HEAD
// Login handles the login of conn giving to the appropriate loginController
+=======
+// Login handles the logout of conn giving to the appropiate loginController
+>>>>>>> 611e6e5fd (container requests in the new codepath for controller)
func (conn *Conn) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
return conn.loginController.Login(ctx, opts)
}
+<<<<<<< HEAD
// UserAuthenticate handles the User Authentication of conn giving to the appropriate loginController
+=======
+// UserAuthenticate handles the User Authentication of conn giving to the appropiate loginController
+>>>>>>> 611e6e5fd (container requests in the new codepath for controller)
func (conn *Conn) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
return conn.loginController.UserAuthenticate(ctx, opts)
}
diff --git a/lib/controller/router/response.go b/lib/controller/router/response.go
index 6c578e0a1..f83afa1dd 100644
--- a/lib/controller/router/response.go
+++ b/lib/controller/router/response.go
@@ -109,6 +109,48 @@ func (rtr *router) sendResponse(w http.ResponseWriter, req *http.Request, resp i
rtr.mungeItemFields(tmp)
}
+<<<<<<< HEAD
+=======
+ for k, v := range tmp {
+ if strings.HasSuffix(k, "_at") {
+ // Format non-nil timestamps as
+ // rfc3339NanoFixed (by default they will have
+ // been encoded to time.RFC3339Nano, which
+ // omits trailing zeroes).
+ switch tv := v.(type) {
+ case *time.Time:
+ if tv == nil {
+ break
+ }
+ tmp[k] = tv.Format(rfc3339NanoFixed)
+ case time.Time:
+ if tv.IsZero() {
+ tmp[k] = nil
+ } else {
+ tmp[k] = tv.Format(rfc3339NanoFixed)
+ }
+ case string:
+ t, err := time.Parse(time.RFC3339Nano, tv)
+ if err != nil {
+ break
+ }
+ tmp[k] = t.Format(rfc3339NanoFixed)
+ }
+ }
+ switch k {
+ // in all this cases, RoR returns nil instead the Zero value for the type.
+ // Maytbe, this should all go away when RoR is out of the picture.
+ case "output_uuid", "output_name", "log_uuid", "modified_by_client_uuid", "description", "requesting_container_uuid", "expires_at":
+ if v == "" {
+ tmp[k] = nil
+ }
+ case "container_count_max":
+ if v == float64(0) {
+ tmp[k] = nil
+ }
+ }
+ }
+>>>>>>> 611e6e5fd (container requests in the new codepath for controller)
w.Header().Set("Content-Type", "application/json")
enc := json.NewEncoder(w)
enc.SetEscapeHTML(false)
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index be95710ee..5ffa66801 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -12,7 +12,6 @@ import (
"errors"
"fmt"
"io"
- "log"
"net"
"net/http"
"net/url"
@@ -288,7 +287,6 @@ func (conn *Conn) ContainerUnlock(ctx context.Context, options arvados.GetOption
}
func (conn *Conn) ContainerRequestCreate(ctx context.Context, options arvados.CreateOptions) (arvados.ContainerRequest, error) {
- log.Printf("THIS IS rcp.Conn.ContainerRequestCreate() for %s we are %s", options.ClusterID, conn.clusterID)
ep := arvados.EndpointContainerRequestCreate
var resp arvados.ContainerRequest
err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index d5f0b5bb1..203b426c9 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -48,7 +48,7 @@ type ContainerRequest struct {
Properties map[string]interface{} `json:"properties"`
State ContainerRequestState `json:"state"`
RequestingContainerUUID string `json:"requesting_container_uuid"`
- ContainerUUID string `json:"container_uuid"`
+ ContainerUUID *string `json:"container_uuid"`
ContainerCountMax int `json:"container_count_max"`
Mounts map[string]Mount `json:"mounts"`
RuntimeConstraints RuntimeConstraints `json:"runtime_constraints"`
commit f437716f309592d9b2ce801c344b82c1ddcd7e39
Author: Ward Vandewege <ward at curii.com>
Date: Thu Dec 10 10:33:57 2020 -0500
Costanalyzer bugfix: when a collection uuid is supplied, write the csv
files with the name of the associated container request uuid, not the
collection uuid. Also change tests to use a proper temporary directory
for the outputs, and do not reuse that directory between tests (which
hid the bug this commit fixed).
refs #17187
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index d81ade607..46ff655dd 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -399,7 +399,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
}
var container arvados.Container
- err = loadObject(logger, ac, uuid, cr.ContainerUUID, cache, &container)
+ err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)
if err != nil {
return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
}
@@ -428,7 +428,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
if err != nil {
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
- logger.Infof("Collecting child containers for container request %s", uuid)
+ logger.Infof("Collecting child containers for container request %s", crUUID)
for _, cr2 := range childCrs.Items {
logger.Info(".")
node, err := getNode(arv, ac, kc, cr2)
@@ -437,7 +437,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
}
logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
var c2 arvados.Container
- err = loadObject(logger, ac, uuid, cr2.ContainerUUID, cache, &c2)
+ err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
if err != nil {
return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
}
@@ -452,7 +452,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
if resultsDir != "" {
// Write the resulting CSV file
- fName := resultsDir + "/" + uuid + ".csv"
+ fName := resultsDir + "/" + crUUID + ".csv"
err = ioutil.WriteFile(fName, []byte(csv), 0644)
if err != nil {
return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 3488f0d8d..b1ddf97a3 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -160,12 +160,13 @@ func (*Suite) TestUsage(c *check.C) {
func (*Suite) TestContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
// Run costanalyzer with 1 container request uuid
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", "results", arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
@@ -180,8 +181,9 @@ func (*Suite) TestContainerRequestUUID(c *check.C) {
func (*Suite) TestCollectionUUID(c *check.C) {
var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
// Run costanalyzer with 1 collection uuid, without 'container_request' property
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", "results", arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 2)
c.Assert(stderr.String(), check.Matches, "(?ms).*does not have a 'container_request' property.*")
@@ -203,11 +205,12 @@ func (*Suite) TestCollectionUUID(c *check.C) {
stderr.Truncate(0)
// Run costanalyzer with 1 collection uuid
- exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", "results", arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
+ resultsDir = c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
@@ -221,16 +224,17 @@ func (*Suite) TestCollectionUUID(c *check.C) {
func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
// Run costanalyzer with 2 container request uuids
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", "results", arvadostest.CompletedContainerRequestUUID, arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedContainerRequestUUID, arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
- uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
@@ -262,15 +266,16 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
c.Assert(err, check.IsNil)
// Run costanalyzer with the project uuid
- exitcode = Command.RunCommand("costanalyzer.test", []string{"-cache=false", "-log-level", "debug", "-output", "results", arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ resultsDir = c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-cache=false", "-log-level", "debug", "-output", resultsDir, arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ uuidReport, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
- uuidReport2, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ uuidReport2, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
@@ -297,15 +302,16 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
stderr.Truncate(0)
// Run costanalyzer with 2 container request uuids
- exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", "results", arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ resultsDir := c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
- uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192")
- uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
commit 82313a238872ed2143679ee18b4a49eef7bd39c1
Author: Tom Clegg <tom at curii.com>
Date: Wed Dec 9 15:44:57 2020 -0500
17202: Test avoiding redirect for cross-origin inline images.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 8e2e05c76..5291efeb8 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -583,6 +583,25 @@ func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
+
+ // GET + Origin header is representative of both AJAX GET
+ // requests and inline images via <IMG crossorigin="anonymous"
+ // src="...">.
+ u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
+ req = &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Origin": {"https://origin.example"},
+ },
+ }
+ resp = httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Equals, "foo")
+ c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
}
func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
commit ff37086ed63714d4eb8f35b34de761b47fe15cde
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Dec 9 09:34:14 2020 -0500
17202: Use explicit SameSite=Lax for 303-with-cookie.
This improves XSS protection on some browsers, including Safari and
Firefox for Android.
On most browsers, Lax is already the default.
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 8e4274038..2d6fb78f8 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -773,6 +773,7 @@ func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, loc
Value: auth.EncodeTokenCookie([]byte(formToken)),
Path: "/",
HttpOnly: true,
+ SameSite: http.SameSiteLaxMode,
})
}
commit 44f0fda05ab9b3cc93205a4365edc0a699fa7957
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Dec 8 19:52:36 2020 -0500
17202: Bypass 303-with-token on cross-origin requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index ab1bc080b..8e4274038 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -296,27 +296,32 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
formToken := r.FormValue("api_token")
- if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
- // The client provided an explicit token in the POST
- // body. The Origin header indicates this *might* be
- // an AJAX request, in which case redirect-with-cookie
- // won't work: we should just serve the content in the
- // POST response. This is safe because:
+ origin := r.Header.Get("Origin")
+ cors := origin != "" && !strings.HasSuffix(origin, "://"+r.Host)
+ safeAjax := cors && (r.Method == http.MethodGet || r.Method == http.MethodHead)
+ safeAttachment := attachment && r.URL.Query().Get("api_token") == ""
+ if formToken == "" {
+ // No token to use or redact.
+ } else if safeAjax || safeAttachment {
+ // If this is a cross-origin request, the URL won't
+ // appear in the browser's address bar, so
+ // substituting a clipboard-safe URL is pointless.
+ // Redirect-with-cookie wouldn't work anyway, because
+ // it's not safe to allow third-party use of our
+ // cookie.
//
- // * We're supplying an attachment, not inline
- // content, so we don't need to convert the POST to
- // a GET and avoid the "really resubmit form?"
- // problem.
- //
- // * The token isn't embedded in the URL, so we don't
- // need to worry about bookmarks and copy/paste.
+ // If we're supplying an attachment, we don't need to
+ // convert POST to GET to avoid the "really resubmit
+ // form?" problem, so provided the token isn't
+ // embedded in the URL, there's no reason to do
+ // redirect-with-cookie in this case either.
reqTokens = append(reqTokens, formToken)
- } else if formToken != "" && browserMethod[r.Method] {
- // The client provided an explicit token in the query
- // string, or a form in POST body. We must put the
- // token in an HttpOnly cookie, and redirect to the
- // same URL with the query param redacted and method =
- // GET.
+ } else if browserMethod[r.Method] {
+ // If this is a page view, and the client provided a
+ // token via query string or POST body, we must put
+ // the token in an HttpOnly cookie, and redirect to an
+ // equivalent URL with the query param redacted and
+ // method = GET.
h.seeOtherWithCookie(w, r, "", credentialsOK)
return
}
commit 0f7fab52b7ca43763bd17fd414009319bd653e51
Author: Ward Vandewege <ward at curii.com>
Date: Wed Dec 9 17:06:06 2020 -0500
Fix ineffassign warning.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index 05ce62e58..0ff897c35 100644
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -351,9 +351,8 @@ func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
c.Check(err, check.IsNil)
c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
- buf, err = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
+ buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
// Command fails because we don't return Etag header.
- // c.Check(err, check.IsNil)
flen := strconv.Itoa(len(testText))
c.Check(string(buf), check.Matches, `(?ms).*`+flen+` of `+flen+`.*`)
}
commit 90a0d1808ec312cd3bd5ebe44edbe2b5fed999db
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Thu Dec 3 17:42:22 2020 -0300
17152: Adds migration to fix collection versions' modified_at timestamps.
Also, with test.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/services/api/db/migrate/20201202174753_fix_collection_versions_timestamps.rb b/services/api/db/migrate/20201202174753_fix_collection_versions_timestamps.rb
new file mode 100644
index 000000000..4c56d3d75
--- /dev/null
+++ b/services/api/db/migrate/20201202174753_fix_collection_versions_timestamps.rb
@@ -0,0 +1,17 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'fix_collection_versions_timestamps'
+
+class FixCollectionVersionsTimestamps < ActiveRecord::Migration[5.2]
+ def up
+ # Defined in a function for easy testing.
+ fix_collection_versions_timestamps
+ end
+
+ def down
+ # This migration is not reversible. However, the results are
+ # backwards compatible.
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 58c064ac3..12a28c6c7 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -3186,6 +3186,7 @@ INSERT INTO "schema_migrations" (version) VALUES
('20200602141328'),
('20200914203202'),
('20201103170213'),
-('20201105190435');
+('20201105190435'),
+('20201202174753');
diff --git a/services/api/lib/fix_collection_versions_timestamps.rb b/services/api/lib/fix_collection_versions_timestamps.rb
new file mode 100644
index 000000000..61da988c0
--- /dev/null
+++ b/services/api/lib/fix_collection_versions_timestamps.rb
@@ -0,0 +1,43 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'set'
+
+include CurrentApiClient
+include ArvadosModelUpdates
+
+def fix_collection_versions_timestamps
+ act_as_system_user do
+ uuids = [].to_set
+ # Get UUIDs from collections with more than 1 version
+ Collection.where(version: 2).find_each(batch_size: 100) do |c|
+ uuids.add(c.current_version_uuid)
+ end
+ uuids.each do |uuid|
+ first_pair = true
+ # All versions of a particular collection get fixed together.
+ ActiveRecord::Base.transaction do
+ Collection.where(current_version_uuid: uuid).order(version: :desc).each_cons(2) do |c1, c2|
+ # Skip if the 2 newest versions' modified_at values are separate enough;
+ # this means that this collection doesn't require fixing, allowing for
+ # migration re-runs in case of transient problems.
+ break if first_pair && (c2.modified_at.to_f - c1.modified_at.to_f) > 1
+ first_pair = false
+ # Fix modified_at timestamps by assigning to N-1's value to N.
+ # Special case: the first version's modified_at will be == to created_at
+ leave_modified_by_user_alone do
+ leave_modified_at_alone do
+ c1.modified_at = c2.modified_at
+ c1.save!(validate: false)
+ if c2.version == 1
+ c2.modified_at = c2.created_at
+ c2.save!(validate: false)
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
\ No newline at end of file
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 767f035b8..61bb3f79f 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -23,8 +23,8 @@ collection_owned_by_active:
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
- modified_at: 2014-02-03T17:22:54Z
- updated_at: 2014-02-03T17:22:54Z
+ modified_at: 2014-02-03T18:22:54Z
+ updated_at: 2014-02-03T18:22:54Z
manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
name: owned_by_active
version: 2
@@ -43,7 +43,7 @@ collection_owned_by_active_with_file_stats:
file_count: 1
file_size_total: 3
name: owned_by_active_with_file_stats
- version: 2
+ version: 1
collection_owned_by_active_past_version_1:
uuid: zzzzz-4zz18-znfnqtbbv4spast
@@ -53,8 +53,8 @@ collection_owned_by_active_past_version_1:
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
- modified_at: 2014-02-03T15:22:54Z
- updated_at: 2014-02-03T15:22:54Z
+ modified_at: 2014-02-03T18:22:54Z
+ updated_at: 2014-02-03T18:22:54Z
manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
name: owned_by_active_version_1
version: 1
@@ -106,8 +106,8 @@ w_a_z_file:
created_at: 2015-02-09T10:53:38Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
- modified_at: 2015-02-09T10:53:38Z
- updated_at: 2015-02-09T10:53:38Z
+ modified_at: 2015-02-09T10:55:38Z
+ updated_at: 2015-02-09T10:55:38Z
manifest_text: ". 4c6c2c0ac8aa0696edd7316a3be5ca3c+5 0:5:w\\040\\141\\040z\n"
name: "\"w a z\" file"
version: 2
@@ -120,8 +120,8 @@ w_a_z_file_version_1:
created_at: 2015-02-09T10:53:38Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
- modified_at: 2015-02-09T10:53:38Z
- updated_at: 2015-02-09T10:53:38Z
+ modified_at: 2015-02-09T10:55:38Z
+ updated_at: 2015-02-09T10:55:38Z
manifest_text: ". 4d20280d5e516a0109768d49ab0f3318+3 0:3:waz\n"
name: "waz file"
version: 1
diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb
index 666d41f0e..a28893e01 100644
--- a/services/api/test/unit/collection_test.rb
+++ b/services/api/test/unit/collection_test.rb
@@ -4,6 +4,7 @@
require 'test_helper'
require 'sweep_trashed_objects'
+require 'fix_collection_versions_timestamps'
class CollectionTest < ActiveSupport::TestCase
include DbCurrentTime
@@ -360,6 +361,29 @@ class CollectionTest < ActiveSupport::TestCase
end
end
+ # Bug #17152 - This test relies on fixtures simulating the problem.
+ test "migration fixing collection versions' modified_at timestamps" do
+ versioned_collection_fixtures = [
+ collections(:w_a_z_file).uuid,
+ collections(:collection_owned_by_active).uuid
+ ]
+ versioned_collection_fixtures.each do |uuid|
+ cols = Collection.where(current_version_uuid: uuid).order(version: :desc)
+ assert_equal cols.size, 2
+ # cols[0] -> head version // cols[1] -> old version
+ assert_operator (cols[0].modified_at.to_f - cols[1].modified_at.to_f), :==, 0
+ assert cols[1].modified_at != cols[1].created_at
+ end
+ fix_collection_versions_timestamps
+ versioned_collection_fixtures.each do |uuid|
+ cols = Collection.where(current_version_uuid: uuid).order(version: :desc)
+ assert_equal cols.size, 2
+ # cols[0] -> head version // cols[1] -> old version
+ assert_operator (cols[0].modified_at.to_f - cols[1].modified_at.to_f), :>, 1
+ assert_operator cols[1].modified_at, :==, cols[1].created_at
+ end
+ end
+
test "past versions should not be directly updatable" do
Rails.configuration.Collections.CollectionVersioning = true
Rails.configuration.Collections.PreserveVersionIfIdle = 0
commit 150f7d25a49f33fa8d2f2b22ea002f573eae2cb2
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Thu Dec 3 15:18:19 2020 -0300
17152: Publishes the arvbox WebDAV download port.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index a180b4363..96f3666cd 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -205,6 +205,7 @@ run() {
--publish=8900:8900
--publish=9000:9000
--publish=9002:9002
+ --publish=9004:9004
--publish=25101:25101
--publish=8001:8001
--publish=8002:8002
commit a8f0e996f2a38995cd97f344b76a692401df10b2
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Tue Dec 1 19:17:01 2020 -0300
17152: Fixes old collection versions' modified_at handling and test.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 8b549a71a..3637f34e1 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -269,6 +269,7 @@ class Collection < ArvadosModel
snapshot = self.dup
snapshot.uuid = nil # Reset UUID so it's created as a new record
snapshot.created_at = self.created_at
+ snapshot.modified_at = self.modified_at_was
end
# Restore requested changes on the current version
@@ -294,8 +295,10 @@ class Collection < ArvadosModel
if snapshot
snapshot.attributes = self.syncable_updates
leave_modified_by_user_alone do
- act_as_system_user do
- snapshot.save
+ leave_modified_at_alone do
+ act_as_system_user do
+ snapshot.save
+ end
end
end
end
diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb
index 48cae5afe..666d41f0e 100644
--- a/services/api/test/unit/collection_test.rb
+++ b/services/api/test/unit/collection_test.rb
@@ -334,6 +334,7 @@ class CollectionTest < ActiveSupport::TestCase
# Set up initial collection
c = create_collection 'foo', Encoding::US_ASCII
assert c.valid?
+ original_version_modified_at = c.modified_at.to_f
# Make changes so that a new version is created
c.update_attributes!({'name' => 'bar'})
c.reload
@@ -344,9 +345,7 @@ class CollectionTest < ActiveSupport::TestCase
version_creation_datetime = c_old.modified_at.to_f
assert_equal c.created_at.to_f, c_old.created_at.to_f
- # Current version is updated just a few milliseconds before the version is
- # saved on the database.
- assert_operator c.modified_at.to_f, :<, version_creation_datetime
+ assert_equal original_version_modified_at, version_creation_datetime
# Make update on current version so old version get the attribute synced;
# its modified_at should not change.
commit 65f0da0b0c8c999274422763d1825f19ccda660e
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Dec 7 13:43:23 2020 -0500
17199: Avoid returning same port twice from find_available_port().
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 0cb4151ac..c79aa4e94 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -73,6 +73,7 @@ if not os.path.exists(TEST_TMPDIR):
my_api_host = None
_cached_config = {}
_cached_db_config = {}
+_already_used_port = {}
def find_server_pid(PID_PATH, wait=10):
now = time.time()
@@ -181,11 +182,15 @@ def find_available_port():
would take care of the races, and this wouldn't be needed at all.
"""
- sock = socket.socket()
- sock.bind(('0.0.0.0', 0))
- port = sock.getsockname()[1]
- sock.close()
- return port
+ global _already_used_port
+ while True:
+ sock = socket.socket()
+ sock.bind(('0.0.0.0', 0))
+ port = sock.getsockname()[1]
+ sock.close()
+ if port not in _already_used_port:
+ _already_used_port[port] = True
+ return port
def _wait_until_port_listens(port, timeout=10, warn=True):
"""Wait for a process to start listening on the given port.
commit fe352f6d7d9b9aa338295f5f3abde0da07131f93
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Dec 2 17:14:37 2020 -0500
17161: Improve SystemRootToken docs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/doc/install/install-api-server.html.textile.liquid b/doc/install/install-api-server.html.textile.liquid
index b8442eb06..c7303bbba 100644
--- a/doc/install/install-api-server.html.textile.liquid
+++ b/doc/install/install-api-server.html.textile.liquid
@@ -51,22 +51,20 @@ h3. Tokens
API:
RailsSessionSecretToken: <span class="userinput">"$rails_secret_token"</span>
Collections:
- BlobSigningKey: <span class="userinput">"blob_signing_key"</span>
+ BlobSigningKey: <span class="userinput">"$blob_signing_key"</span>
</code></pre>
</notextile>
- at SystemRootToken@ is used by Arvados system services to authenticate as the system (root) user when communicating with the API server.
+These secret tokens are used to authenticate messages between Arvados components.
+* @SystemRootToken@ is used by Arvados system services to authenticate as the system (root) user when communicating with the API server.
+* @ManagementToken@ is used to authenticate access to system metrics.
+* @API.RailsSessionSecretToken@ is used to sign session cookies.
+* @Collections.BlobSigningKey@ is used to control access to Keep blocks.
- at ManagementToken@ is used to authenticate access to system metrics.
-
- at API.RailsSessionSecretToken@ is required by the API server.
-
- at Collections.BlobSigningKey@ is used to control access to Keep blocks.
-
-You can generate a random token for each of these items at the command line like this:
+Each token should be a string of at least 50 alphanumeric characters. You can generate a suitable token with the following command:
<notextile>
-<pre><code>~$ <span class="userinput">tr -dc 0-9a-zA-Z </dev/urandom | head -c50; echo</span>
+<pre><code>~$ <span class="userinput">tr -dc 0-9a-zA-Z </dev/urandom | head -c50 ; echo</span>
</code></pre>
</notextile>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 005d2738d..7e16688d9 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -12,6 +12,8 @@
Clusters:
xxxxx:
+ # Token used internally by Arvados components to authenticate to
+ # one another. Use a string of at least 50 random alphanumerics.
SystemRootToken: ""
# Token to be included in all healthcheck requests. Disabled by default.
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 885bb4b8c..934131bd8 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -18,6 +18,8 @@ var DefaultYAML = []byte(`# Copyright (C) The Arvados Authors. All rights reserv
Clusters:
xxxxx:
+ # Token used internally by Arvados components to authenticate to
+ # one another. Use a string of at least 50 random alphanumerics.
SystemRootToken: ""
# Token to be included in all healthcheck requests. Disabled by default.
commit d5efe792b79cd736e68efab107887ae28b3cf0b6
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Dec 2 17:36:35 2020 -0500
17009: Fix s3 ListObjects endpoint with vhost-style requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 7fb90789a..97201d292 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -249,11 +249,14 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
var objectNameGiven bool
+ var bucketName string
fspath := "/by_id"
if id := parseCollectionIDFromDNSName(r.Host); id != "" {
fspath += "/" + id
+ bucketName = id
objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
} else {
+ bucketName = strings.SplitN(strings.TrimPrefix(r.URL.Path, "/"), "/", 2)[0]
objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
}
fspath += r.URL.Path
@@ -268,7 +271,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
fmt.Fprintln(w, `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`)
} else {
// ListObjects
- h.s3list(w, r, fs)
+ h.s3list(bucketName, w, r, fs)
}
return true
case r.Method == http.MethodGet || r.Method == http.MethodHead:
@@ -504,15 +507,13 @@ func walkFS(fs arvados.CustomFileSystem, path string, isRoot bool, fn func(path
var errDone = errors.New("done")
-func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
+func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
var params struct {
- bucket string
delimiter string
marker string
maxKeys int
prefix string
}
- params.bucket = strings.SplitN(r.URL.Path[1:], "/", 2)[0]
params.delimiter = r.FormValue("delimiter")
params.marker = r.FormValue("marker")
if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 && mk < s3MaxKeys {
@@ -522,7 +523,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
}
params.prefix = r.FormValue("prefix")
- bucketdir := "by_id/" + params.bucket
+ bucketdir := "by_id/" + bucket
// walkpath is the directory (relative to bucketdir) we need
// to walk: the innermost directory that is guaranteed to
// contain all paths that have the requested prefix. Examples:
@@ -557,7 +558,7 @@ func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.Cust
}
resp := listResp{
ListResp: s3.ListResp{
- Name: strings.SplitN(r.URL.Path[1:], "/", 2)[0],
+ Name: bucket,
Prefix: params.prefix,
Delimiter: params.delimiter,
Marker: params.marker,
commit c18025fcc5c2c782a35e7c9c582489be623e671a
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Dec 3 16:24:37 2020 -0500
17009: Test virtual host-style S3 requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index 562d5296b..a6aab357e 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -10,6 +10,7 @@ import (
"fmt"
"io/ioutil"
"net/http"
+ "net/http/httptest"
"net/url"
"os"
"os/exec"
@@ -440,6 +441,70 @@ func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
c.Assert(fs.Sync(), check.IsNil)
}
+func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+ for _, trial := range []struct {
+ url string
+ method string
+ body string
+ responseCode int
+ responseRegexp []string
+ }{
+ {
+ url: "https://" + stage.collbucket.Name + ".example.com/",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+ },
+ {
+ url: "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`⛵\n`},
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
+ method: "PUT",
+ body: "boop",
+ responseCode: http.StatusOK,
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`boop`},
+ },
+ } {
+ url, err := url.Parse(trial.url)
+ c.Assert(err, check.IsNil)
+ req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+ rr := httptest.NewRecorder()
+ s.testServer.Server.Handler.ServeHTTP(rr, req)
+ resp := rr.Result()
+ c.Check(resp.StatusCode, check.Equals, trial.responseCode)
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Assert(err, check.IsNil)
+ for _, re := range trial.responseRegexp {
+ c.Check(string(body), check.Matches, re)
+ }
+ }
+}
+
func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
commit cdee3de0f2cd7d8169fbc6cc052c0db4fcfc9369
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Dec 7 14:20:27 2020 -0500
Fix tests.
refs #17022
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/api/test/unit/create_superuser_token_test.rb b/services/api/test/unit/create_superuser_token_test.rb
index e95e0f226..3c6dcbdbb 100644
--- a/services/api/test/unit/create_superuser_token_test.rb
+++ b/services/api/test/unit/create_superuser_token_test.rb
@@ -9,11 +9,11 @@ require 'create_superuser_token'
class CreateSuperUserTokenTest < ActiveSupport::TestCase
include CreateSuperUserToken
- test "create superuser token twice and expect same resutls" do
+ test "create superuser token twice and expect same results" do
# Create a token with some string
token1 = create_superuser_token 'atesttoken'
assert_not_nil token1
- assert_equal token1, 'atesttoken'
+ assert_match(/atesttoken$/, token1)
# Create token again; this time, we should get the one created earlier
token2 = create_superuser_token
@@ -25,7 +25,7 @@ class CreateSuperUserTokenTest < ActiveSupport::TestCase
# Create a token with some string
token1 = create_superuser_token 'atesttoken'
assert_not_nil token1
- assert_equal token1, 'atesttoken'
+ assert_match(/\/atesttoken$/, token1)
# Create token again with some other string and expect the existing superuser token back
token2 = create_superuser_token 'someothertokenstring'
@@ -33,37 +33,26 @@ class CreateSuperUserTokenTest < ActiveSupport::TestCase
assert_equal token1, token2
end
- test "create superuser token twice and expect same results" do
- # Create a token with some string
- token1 = create_superuser_token 'atesttoken'
- assert_not_nil token1
- assert_equal token1, 'atesttoken'
-
- # Create token again with that same superuser token and expect it back
- token2 = create_superuser_token 'atesttoken'
- assert_not_nil token2
- assert_equal token1, token2
- end
-
test "create superuser token and invoke again with some other valid token" do
# Create a token with some string
token1 = create_superuser_token 'atesttoken'
assert_not_nil token1
- assert_equal token1, 'atesttoken'
+ assert_match(/\/atesttoken$/, token1)
su_token = api_client_authorizations("system_user").api_token
token2 = create_superuser_token su_token
- assert_equal token2, su_token
+ assert_equal token2.split('/')[2], su_token
end
test "create superuser token, expire it, and create again" do
# Create a token with some string
token1 = create_superuser_token 'atesttoken'
assert_not_nil token1
- assert_equal token1, 'atesttoken'
+ assert_match(/\/atesttoken$/, token1)
# Expire this token and call create again; expect a new token created
- apiClientAuth = ApiClientAuthorization.where(api_token: token1).first
+ apiClientAuth = ApiClientAuthorization.where(api_token: 'atesttoken').first
+ refute_nil apiClientAuth
Thread.current[:user] = users(:admin)
apiClientAuth.update_attributes expires_at: '2000-10-10'
commit 7919e888b7028b803ae7ab9cd4adfae3fe413d63
Author: Javier Bértoli <jbertoli at curii.com>
Date: Mon Dec 7 12:46:06 2020 -0300
fix(provision): update arvados-formula's version
refs #17177
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index ab239250b..a4d55c221 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -49,7 +49,7 @@ VERSION="latest"
# Usually there's no need to modify things below this line
# Formulas versions
-ARVADOS_TAG="v1.1.2"
+ARVADOS_TAG="v1.1.3"
POSTGRES_TAG="v0.41.3"
NGINX_TAG="v2.4.0"
DOCKER_TAG="v1.0.0"
commit d0301d9ddbf002fc227bbb56274e7e0446187a6f
Author: Javier Bértoli <jbertoli at curii.com>
Date: Mon Dec 7 11:22:08 2020 -0300
fix(provision): pin arvados-formula
refs #17177
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 3fa233b61..ab239250b 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -49,6 +49,7 @@ VERSION="latest"
# Usually there's no need to modify things below this line
# Formulas versions
+ARVADOS_TAG="v1.1.2"
POSTGRES_TAG="v0.41.3"
NGINX_TAG="v2.4.0"
DOCKER_TAG="v1.0.0"
@@ -190,7 +191,7 @@ EOFPSLS
# Get the formula and dependencies
cd ${F_DIR} || exit 1
-git clone https://github.com/netmanagers/arvados-formula.git
+git clone --branch "${ARVADOS_TAG}" https://github.com/saltstack-formulas/arvados-formula.git
git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
commit 08dcbff16dca8ebfd5665afc8850c04ca6fee51e
Author: Javier Bértoli <jbertoli at curii.com>
Date: Mon Dec 7 09:20:29 2020 -0300
fix(provision): add port to workbench2 nginx's stanza
refs #17177
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/.gitignore b/.gitignore
index 877ccdf4d..beb84b3c2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,3 +32,5 @@ services/api/config/arvados-clients.yml
.Rproj.user
_version.py
*.bak
+arvados-snakeoil-ca.pem
+.vagrant
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 82a0a4a93..3fa233b61 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -190,7 +190,7 @@ EOFPSLS
# Get the formula and dependencies
cd ${F_DIR} || exit 1
-git clone https://github.com/saltstack-formulas/arvados-formula.git
+git clone https://github.com/netmanagers/arvados-formula.git
git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
diff --git a/tools/salt-install/single_host/nginx_workbench2_configuration.sls b/tools/salt-install/single_host/nginx_workbench2_configuration.sls
index d30b13382..8930be408 100644
--- a/tools/salt-install/single_host/nginx_workbench2_configuration.sls
+++ b/tools/salt-install/single_host/nginx_workbench2_configuration.sls
@@ -42,7 +42,7 @@ nginx:
- 'if (-f $document_root/maintenance.html)':
- return: 503
- location /config.json:
- - return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__"}' ~ "'" }}
+ - return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__"}' ~ "'" }}
- include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
commit 799bde4ed30dbd73fbbd0b59dc6d6d9f900dea8a
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Dec 4 13:22:38 2020 -0300
fix(provision): pin formulas' versions
* Pin formulas versions, to prevent changes upstream breaking the installer
* Remove verbose/debug flags
* Remove unused entries in the config pillars
refs #17177
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index 1f587296b..6966ea834 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -33,7 +33,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
arv.vm.provision "shell",
path: "provision.sh",
args: [
- "--debug",
+ # "--debug",
"--test",
"--vagrant",
"--ssl-port=8443"
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 7b4fc9da3..82a0a4a93 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash
# Copyright (C) The Arvados Authors. All rights reserved.
#
@@ -48,6 +48,12 @@ VERSION="latest"
##########################################################
# Usually there's no need to modify things below this line
+# Formulas versions
+POSTGRES_TAG="v0.41.3"
+NGINX_TAG="v2.4.0"
+DOCKER_TAG="v1.0.0"
+LOCALE_TAG="v0.3.4"
+
set -o pipefail
# capture the directory that the script is running from
@@ -184,9 +190,11 @@ EOFPSLS
# Get the formula and dependencies
cd ${F_DIR} || exit 1
-for f in postgres arvados nginx docker locale; do
- git clone https://github.com/saltstack-formulas/${f}-formula.git
-done
+git clone https://github.com/saltstack-formulas/arvados-formula.git
+git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
+git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
+git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
+git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
if [ "x${BRANCH}" != "x" ]; then
cd ${F_DIR}/arvados-formula || exit 1
diff --git a/tools/salt-install/single_host/nginx_controller_configuration.sls b/tools/salt-install/single_host/nginx_controller_configuration.sls
index 96fc383d7..00c3b3a13 100644
--- a/tools/salt-install/single_host/nginx_controller_configuration.sls
+++ b/tools/salt-install/single_host/nginx_controller_configuration.sls
@@ -52,7 +52,6 @@ nginx:
- proxy_set_header: 'X-Real-IP $remote_addr'
- proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
- proxy_set_header: 'X-External-Client $external_client'
- # - include: 'snippets/letsencrypt.conf'
- include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_keepproxy_configuration.sls b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
index 61c138474..6554f79a7 100644
--- a/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
+++ b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
@@ -52,7 +52,6 @@ nginx:
- client_max_body_size: 64M
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- # - include: 'snippets/letsencrypt.conf'
- include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_keepweb_configuration.sls b/tools/salt-install/single_host/nginx_keepweb_configuration.sls
index 88083e3c5..cc871b9da 100644
--- a/tools/salt-install/single_host/nginx_keepweb_configuration.sls
+++ b/tools/salt-install/single_host/nginx_keepweb_configuration.sls
@@ -52,7 +52,6 @@ nginx:
- client_max_body_size: 0
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- # - include: 'snippets/letsencrypt.conf'
- include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_webshell_configuration.sls b/tools/salt-install/single_host/nginx_webshell_configuration.sls
index 80e9f57d6..a0756b7ce 100644
--- a/tools/salt-install/single_host/nginx_webshell_configuration.sls
+++ b/tools/salt-install/single_host/nginx_webshell_configuration.sls
@@ -68,7 +68,6 @@ nginx:
- add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
- add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
- # - include: 'snippets/letsencrypt.conf'
- include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_websocket_configuration.sls b/tools/salt-install/single_host/nginx_websocket_configuration.sls
index 60d757f89..ebe03f733 100644
--- a/tools/salt-install/single_host/nginx_websocket_configuration.sls
+++ b/tools/salt-install/single_host/nginx_websocket_configuration.sls
@@ -53,7 +53,6 @@ nginx:
- client_max_body_size: 64M
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
- # - include: 'snippets/letsencrypt.conf'
- include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_workbench2_configuration.sls b/tools/salt-install/single_host/nginx_workbench2_configuration.sls
index 4a0190ad1..d30b13382 100644
--- a/tools/salt-install/single_host/nginx_workbench2_configuration.sls
+++ b/tools/salt-install/single_host/nginx_workbench2_configuration.sls
@@ -43,7 +43,6 @@ nginx:
- return: 503
- location /config.json:
- return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__"}' ~ "'" }}
- # - include: 'snippets/letsencrypt.conf'
- include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_workbench_configuration.sls b/tools/salt-install/single_host/nginx_workbench_configuration.sls
index 6a17ee745..be571ca77 100644
--- a/tools/salt-install/single_host/nginx_workbench_configuration.sls
+++ b/tools/salt-install/single_host/nginx_workbench_configuration.sls
@@ -54,7 +54,6 @@ nginx:
- proxy_set_header: 'Host $http_host'
- proxy_set_header: 'X-Real-IP $remote_addr'
- proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
- # - include: 'snippets/letsencrypt.conf'
- include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
commit 7e4e9c15e6416115f53e511eb121c4d667bca61e
Author: Javier Bértoli <jbertoli at curii.com>
Date: Fri Dec 4 09:17:34 2020 -0300
fix(provision): Document CA certificate purpose and installation
refs #17177
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/doc/install/arvbox.html.textile.liquid b/doc/install/arvbox.html.textile.liquid
index c01ec61fa..3c77ade8d 100644
--- a/doc/install/arvbox.html.textile.liquid
+++ b/doc/install/arvbox.html.textile.liquid
@@ -80,10 +80,23 @@ Arvbox creates root certificate to authorize Arvbox services. Installing the ro
The certificate will be added under the "Arvados testing" organization as "arvbox testing root CA".
-To access your Arvbox instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage (instructions for Debian/Ubuntu):
+To access your Arvbox instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage.
-# copy @arvbox-root-cert.pem@ to @/usr/local/share/ca-certificates/@
-# run @/usr/sbin/update-ca-certificates@
+h3. On Debian/Ubuntu:
+
+<notextile>
+<pre><code>cp arvbox-root-cert.pem /usr/local/share/ca-certificates/
+/usr/sbin/update-ca-certificates
+</code></pre>
+</notextile>
+
+h3. On CentOS:
+
+<notextile>
+<pre><code>cp arvbox-root-cert.pem /etc/pki/ca-trust/source/anchors/
+/usr/bin/update-ca-trust
+</code></pre>
+</notextile>
h2. Configs
diff --git a/doc/install/salt-single-host.html.textile.liquid b/doc/install/salt-single-host.html.textile.liquid
index fb41d59ee..5bed6d05e 100644
--- a/doc/install/salt-single-host.html.textile.liquid
+++ b/doc/install/salt-single-host.html.textile.liquid
@@ -11,7 +11,9 @@ SPDX-License-Identifier: CC-BY-SA-3.0
# "Install Saltstack":#saltstack
# "Single host install using the provision.sh script":#single_host
-# "DNS configuration":#final_steps
+# "Final steps":#final_steps
+## "DNS configuration":#dns_configuration
+## "Install root certificate":#ca_root_certificate
# "Initial user and login":#initial_user
# "Test the installed cluster running a simple workflow":#test_install
@@ -49,7 +51,9 @@ arvados: Failed: 0
</code></pre>
</notextile>
-h2(#final_steps). DNS configuration
+h2(#final_steps). Final configuration steps
+
+h3(#dns_configuration). DNS configuration
After the setup is done, you need to set up your DNS to be able to access the cluster.
@@ -65,6 +69,39 @@ echo "${HOST_IP} api keep keep0 collections download ws workbench workbench2 ${C
</code></pre>
</notextile>
+h3(#ca_root_certificate). Install root certificate
+
+Arvados uses SSL to encrypt communications. Its UI uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
+
+For this reason, the @arvados-formula@ has a helper state to create a root certificate to authorize Arvados services. The @provision.sh@ script will leave a copy of the generated CA's certificate (@arvados-snakeoil-ca.pem@) in the script's directory so ypu can add it to your workstation.
+
+Installing the root certificate into your web browser will prevent security errors when accessing Arvados services with your web browser.
+
+# Go to the certificate manager in your browser.
+#* In Chrome, this can be found under "Settings → Advanced → Manage Certificates" or by entering @chrome://settings/certificates@ in the URL bar.
+#* In Firefox, this can be found under "Preferences → Privacy & Security" or entering @about:preferences#privacy@ in the URL bar and then choosing "View Certificates...".
+# Select the "Authorities" tab, then press the "Import" button. Choose @arvados-snakeoil-ca.pem@
+
+The certificate will be added under the "Arvados Formula".
+
+To access your Arvados instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage.
+
+* On Debian/Ubuntu:
+
+<notextile>
+<pre><code>cp arvados-root-cert.pem /usr/local/share/ca-certificates/
+/usr/sbin/update-ca-certificates
+</code></pre>
+</notextile>
+
+* On CentOS:
+
+<notextile>
+<pre><code>cp arvados-root-cert.pem /etc/pki/ca-trust/source/anchors/
+/usr/bin/update-ca-trust
+</code></pre>
+</notextile>
+
h2(#initial_user). Initial user and login
At this point you should be able to log into the Arvados cluster.
diff --git a/doc/install/salt-vagrant.html.textile.liquid b/doc/install/salt-vagrant.html.textile.liquid
index d9aa791f0..ed0d5bca6 100644
--- a/doc/install/salt-vagrant.html.textile.liquid
+++ b/doc/install/salt-vagrant.html.textile.liquid
@@ -10,7 +10,9 @@ SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
# "Vagrant":#vagrant
-# "DNS configuration":#final_steps
+# "Final steps":#final_steps
+## "DNS configuration":#dns_configuration
+## "Install root certificate":#ca_root_certificate
# "Initial user and login":#initial_user
# "Test the installed cluster running a simple workflow":#test_install
@@ -37,7 +39,9 @@ If you want to reconfigure the running box, you can just:
</code></pre>
</notextile>
-h2(#final_steps). DNS configuration
+h2(#final_steps). Final configuration steps
+
+h3(#dns_configuration). DNS configuration
After the setup is done, you need to set up your DNS to be able to access the cluster.
@@ -53,6 +57,39 @@ echo "${HOST_IP} api keep keep0 collections download ws workbench workbench2 ${C
</code></pre>
</notextile>
+h3(#ca_root_certificate). Install root certificate
+
+Arvados uses SSL to encrypt communications. Its UI uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
+
+For this reason, the @arvados-formula@ has a helper state to create a root certificate to authorize Arvados services. The @provision.sh@ script will leave a copy of the generated CA's certificate (@arvados-snakeoil-ca.pem@) in the script's directory so ypu can add it to your workstation.
+
+Installing the root certificate into your web browser will prevent security errors when accessing Arvados services with your web browser.
+
+# Go to the certificate manager in your browser.
+#* In Chrome, this can be found under "Settings → Advanced → Manage Certificates" or by entering @chrome://settings/certificates@ in the URL bar.
+#* In Firefox, this can be found under "Preferences → Privacy & Security" or entering @about:preferences#privacy@ in the URL bar and then choosing "View Certificates...".
+# Select the "Authorities" tab, then press the "Import" button. Choose @arvados-snakeoil-ca.pem@
+
+The certificate will be added under the "Arvados Formula".
+
+To access your Arvados instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage.
+
+* On Debian/Ubuntu:
+
+<notextile>
+<pre><code>cp arvados-root-cert.pem /usr/local/share/ca-certificates/
+/usr/sbin/update-ca-certificates
+</code></pre>
+</notextile>
+
+* On CentOS:
+
+<notextile>
+<pre><code>cp arvados-root-cert.pem /etc/pki/ca-trust/source/anchors/
+/usr/bin/update-ca-trust
+</code></pre>
+</notextile>
+
h2(#initial_user). Initial user and login
At this point you should be able to log into the Arvados cluster.
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 9aa5f19b1..7b4fc9da3 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -258,7 +258,7 @@ fi
# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
# Leave a copy of the Arvados CA so the user can copy it where it's required
-echo "Copying the Arvados CA file to the installer dir, so you can import it"
+echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
# If running in a vagrant VM, also add default user to docker group
if [ "x${VAGRANT}" = "xyes" ]; then
cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant
commit 7f84cbbc200a15c08a35c341af413f6944b4ce6d
Author: Javier Bértoli <jbertoli at curii.com>
Date: Thu Dec 3 20:00:46 2020 -0300
fix(provision): Add a CA and sign certificates with it
refs #17177
As discussed [here](https://forum.arvados.org/t/debugging-arvados-deployed-with-salt/58/8)
and [here](https://gitter.im/arvados/community?at=5fc65683496ca3372e3474a3), Arvados needs
certs signed by a known CA to work correctly.
This PR adds a CA and leaves a copy of the certificate in the installer directory.
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index ed3466dde..1f587296b 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -33,6 +33,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
arv.vm.provision "shell",
path: "provision.sh",
args: [
+ "--debug",
"--test",
"--vagrant",
"--ssl-port=8443"
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index a207d0198..9aa5f19b1 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
# Copyright (C) The Arvados Authors. All rights reserved.
#
@@ -139,7 +139,7 @@ file_roots:
base:
- ${S_DIR}
- ${F_DIR}/*
- - ${F_DIR}/*/test/salt/states
+ - ${F_DIR}/*/test/salt/states/examples
pillar_roots:
base:
@@ -154,8 +154,8 @@ mkdir -p ${P_DIR}
cat > ${S_DIR}/top.sls << EOFTSLS
base:
'*':
- - example_single_host_host_entries
- - example_add_snakeoil_certs
+ - single_host.host_entries
+ - single_host.snakeoil_certs
- locale
- nginx.passenger
- postgres
@@ -182,7 +182,6 @@ base:
- postgresql
EOFPSLS
-
# Get the formula and dependencies
cd ${F_DIR} || exit 1
for f in postgres arvados nginx docker locale; do
@@ -258,9 +257,16 @@ if [ "x${RESTORE_PSQL}" = "xyes" ]; then
fi
# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
-# If running in a vagrant VM, add default user to docker group
+# Leave a copy of the Arvados CA so the user can copy it where it's required
+echo "Copying the Arvados CA file to the installer dir, so you can import it"
+# If running in a vagrant VM, also add default user to docker group
if [ "x${VAGRANT}" = "xyes" ]; then
- usermod -a -G docker vagrant
+ cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant
+
+ echo "Adding the vagrant user to the docker group"
+ usermod -a -G docker vagrant
+else
+ cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}
fi
# Test that the installation finished correctly
diff --git a/tools/salt-install/single_host/arvados.sls b/tools/salt-install/single_host/arvados.sls
index dffd6575e..a06244270 100644
--- a/tools/salt-install/single_host/arvados.sls
+++ b/tools/salt-install/single_host/arvados.sls
@@ -73,7 +73,7 @@ arvados:
tls:
# certificate: ''
# key: ''
- # required to test with snakeoil certs
+ # required to test with arvados-snakeoil certs
insecure: true
### TOKENS
diff --git a/tools/salt-install/single_host/nginx_controller_configuration.sls b/tools/salt-install/single_host/nginx_controller_configuration.sls
index 7c99d2dea..96fc383d7 100644
--- a/tools/salt-install/single_host/nginx_controller_configuration.sls
+++ b/tools/salt-install/single_host/nginx_controller_configuration.sls
@@ -53,7 +53,7 @@ nginx:
- proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
- proxy_set_header: 'X-External-Client $external_client'
# - include: 'snippets/letsencrypt.conf'
- - include: 'snippets/snakeoil.conf'
+ - include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
- client_max_body_size: 128m
diff --git a/tools/salt-install/single_host/nginx_keepproxy_configuration.sls b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
index fc4854e5a..61c138474 100644
--- a/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
+++ b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
@@ -53,6 +53,6 @@ nginx:
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
# - include: 'snippets/letsencrypt.conf'
- - include: 'snippets/snakeoil.conf'
+ - include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_keepweb_configuration.sls b/tools/salt-install/single_host/nginx_keepweb_configuration.sls
index 513c0393e..88083e3c5 100644
--- a/tools/salt-install/single_host/nginx_keepweb_configuration.sls
+++ b/tools/salt-install/single_host/nginx_keepweb_configuration.sls
@@ -53,6 +53,6 @@ nginx:
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
# - include: 'snippets/letsencrypt.conf'
- - include: 'snippets/snakeoil.conf'
+ - include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_webshell_configuration.sls b/tools/salt-install/single_host/nginx_webshell_configuration.sls
index 495de82d2..80e9f57d6 100644
--- a/tools/salt-install/single_host/nginx_webshell_configuration.sls
+++ b/tools/salt-install/single_host/nginx_webshell_configuration.sls
@@ -69,7 +69,7 @@ nginx:
- add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
# - include: 'snippets/letsencrypt.conf'
- - include: 'snippets/snakeoil.conf'
+ - include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_websocket_configuration.sls b/tools/salt-install/single_host/nginx_websocket_configuration.sls
index 1848a8737..60d757f89 100644
--- a/tools/salt-install/single_host/nginx_websocket_configuration.sls
+++ b/tools/salt-install/single_host/nginx_websocket_configuration.sls
@@ -54,6 +54,6 @@ nginx:
- proxy_http_version: '1.1'
- proxy_request_buffering: 'off'
# - include: 'snippets/letsencrypt.conf'
- - include: 'snippets/snakeoil.conf'
+ - include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_workbench2_configuration.sls b/tools/salt-install/single_host/nginx_workbench2_configuration.sls
index 733397adf..4a0190ad1 100644
--- a/tools/salt-install/single_host/nginx_workbench2_configuration.sls
+++ b/tools/salt-install/single_host/nginx_workbench2_configuration.sls
@@ -44,6 +44,6 @@ nginx:
- location /config.json:
- return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__"}' ~ "'" }}
# - include: 'snippets/letsencrypt.conf'
- - include: 'snippets/snakeoil.conf'
+ - include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_workbench_configuration.sls b/tools/salt-install/single_host/nginx_workbench_configuration.sls
index 9a382e777..6a17ee745 100644
--- a/tools/salt-install/single_host/nginx_workbench_configuration.sls
+++ b/tools/salt-install/single_host/nginx_workbench_configuration.sls
@@ -55,7 +55,7 @@ nginx:
- proxy_set_header: 'X-Real-IP $remote_addr'
- proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
# - include: 'snippets/letsencrypt.conf'
- - include: 'snippets/snakeoil.conf'
+ - include: 'snippets/arvados-snakeoil.conf'
- access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
- error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
index cf61d92b5..8d9de6fdf 100755
--- a/tools/salt-install/tests/run-test.sh
+++ b/tools/salt-install/tests/run-test.sh
@@ -7,6 +7,15 @@ export ARVADOS_API_TOKEN=changemesystemroottoken
export ARVADOS_API_HOST=__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
export ARVADOS_API_HOST_INSECURE=true
+set -o pipefail
+
+# First, validate that the CA is installed and that we can query it with no errors.
+if ! curl -s -o /dev/null https://workbench.${ARVADOS_API_HOST}/users/welcome?return_to=%2F; then
+ echo "The Arvados CA was not correctly installed. Although some components will work,"
+ echo "others won't. Please verify that the CA cert file was installed correctly and"
+ echo "retry running these tests."
+ exit 1
+fi
# https://doc.arvados.org/v2.0/install/install-jobs-image.html
echo "Creating Arvados Standard Docker Images project"
commit f3bc4d4bf4771cb2b4ea5cf2db822e4c21f5820b
Author: Ward Vandewege <ward at curii.com>
Date: Mon Dec 7 19:36:54 2020 -0500
17187: implement review feedback:
* uuids are now specified at the end of the option list
* invalid uuids will cause the command to error out
* output directory is now optional
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index bca23b153..d81ade607 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -56,12 +56,12 @@ func parseFlags(prog string, args []string, loader *config.Loader, logger *logru
flags.Usage = func() {
fmt.Fprintf(flags.Output(), `
Usage:
- %s [options ...]
+ %s [options ...] <uuid> ...
This program analyzes the cost of Arvados container requests. For each uuid
supplied, it creates a CSV report that lists all the containers used to
fulfill the container request, together with the machine type and cost of
- each container.
+ each container. At least one uuid must be specified.
When supplied with the uuid of a container request, it will calculate the
cost of that container request and all its children.
@@ -97,13 +97,15 @@ Usage:
This program prints the total dollar amount from the aggregate cost
accounting across all provided uuids on stdout.
+ When the '-output' option is specified, a set of CSV files with cost details
+ will be written to the provided directory.
+
Options:
`, prog)
flags.PrintDefaults()
}
loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
- flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports (required)")
- flags.Var(&uuids, "uuid", "object uuid. May be specified more than once. Also accepts a comma separated list of uuids (required)")
+ flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
err = flags.Parse(args)
if err == flag.ErrHelp {
@@ -114,6 +116,7 @@ Options:
exitCode = 2
return
}
+ uuids = flags.Args()
if len(uuids) < 1 {
flags.Usage()
@@ -122,13 +125,6 @@ Options:
return
}
- if resultsDir == "" {
- flags.Usage()
- err = fmt.Errorf("Error: output directory must be specified")
- exitCode = 2
- return
- }
-
lvl, err := logrus.ParseLevel(*loglevel)
if err != nil {
exitCode = 2
@@ -390,7 +386,10 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
if !ok {
return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid)
}
- crUUID = value.(string)
+ crUUID, ok = value.(string)
+ if !ok {
+ return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property of the string type", uuid)
+ }
}
// This is a container request, find the container
@@ -451,13 +450,15 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
- // Write the resulting CSV file
- fName := resultsDir + "/" + uuid + ".csv"
- err = ioutil.WriteFile(fName, []byte(csv), 0644)
- if err != nil {
- return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+ if resultsDir != "" {
+ // Write the resulting CSV file
+ fName := resultsDir + "/" + uuid + ".csv"
+ err = ioutil.WriteFile(fName, []byte(csv), 0644)
+ if err != nil {
+ return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+ }
+ logger.Infof("\nUUID report in %s\n\n", fName)
}
- logger.Infof("\nUUID report in %s\n\n", fName)
return
}
@@ -467,10 +468,12 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
if exitcode != 0 {
return
}
- err = ensureDirectory(logger, resultsDir)
- if err != nil {
- exitcode = 3
- return
+ if resultsDir != "" {
+ err = ensureDirectory(logger, resultsDir)
+ if err != nil {
+ exitcode = 3
+ return
+ }
}
// Arvados Client setup
@@ -519,6 +522,10 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
// "Home" project is not supported by this program. Skip this uuid, but
// keep going.
logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
+ } else {
+ logger.Errorf("This argument does not look like a uuid: %s\n", uuid)
+ exitcode = 3
+ return
}
}
@@ -542,15 +549,18 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
- // Write the resulting CSV file
- aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
- err = ioutil.WriteFile(aFile, []byte(csv), 0644)
- if err != nil {
- err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error())
- exitcode = 1
- return
+ if resultsDir != "" {
+ // Write the resulting CSV file
+ aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+ err = ioutil.WriteFile(aFile, []byte(csv), 0644)
+ if err != nil {
+ err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error())
+ exitcode = 1
+ return
+ }
+ logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
}
- logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
+
// Output the total dollar amount on stdout
fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64))
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 48e7733e1..3488f0d8d 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -161,7 +161,7 @@ func (*Suite) TestUsage(c *check.C) {
func (*Suite) TestContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 1 container request uuid
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", "results", arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
@@ -181,7 +181,7 @@ func (*Suite) TestCollectionUUID(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 1 collection uuid, without 'container_request' property
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.FooCollection, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", "results", arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 2)
c.Assert(stderr.String(), check.Matches, "(?ms).*does not have a 'container_request' property.*")
@@ -203,7 +203,7 @@ func (*Suite) TestCollectionUUID(c *check.C) {
stderr.Truncate(0)
// Run costanalyzer with 1 collection uuid
- exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.FooCollection, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", "results", arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
@@ -222,7 +222,7 @@ func (*Suite) TestCollectionUUID(c *check.C) {
func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 2 container request uuids
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", "results", arvadostest.CompletedContainerRequestUUID, arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
@@ -262,7 +262,7 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
c.Assert(err, check.IsNil)
// Run costanalyzer with the project uuid
- exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID, "-cache=false", "-log-level", "debug", "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-cache=false", "-log-level", "debug", "-output", "results", arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
@@ -285,10 +285,10 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
var stdout, stderr bytes.Buffer
- // Run costanalyzer with 2 container request uuids, as one comma separated -uuid argument
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID + "," + arvadostest.CompletedDiagnosticsContainerRequest2UUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ // Run costanalyzer with 2 container request uuids, without output directory specified
+ exitcode := Command.RunCommand("costanalyzer.test", []string{arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Not(check.Matches), "(?ms).*supplied uuids in .*")
// Check that the total amount was printed to stdout
c.Check(stdout.String(), check.Matches, "0.01492030\n")
@@ -297,7 +297,7 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
stderr.Truncate(0)
// Run costanalyzer with 2 container request uuids
- exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", "results", arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
commit 7b0ef99243c37816540a09670f7782c681cfb760
Author: Ward Vandewege <ward at curii.com>
Date: Sun Dec 6 10:54:11 2020 -0500
17187: costanalyzer: print the total from the aggregate cost accounting
across all uuids to stdout.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index 01a9d9fdb..bca23b153 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -94,6 +94,9 @@ Usage:
In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
+ This program prints the total dollar amount from the aggregate cost
+ accounting across all provided uuids on stdout.
+
Options:
`, prog)
flags.PrintDefaults()
@@ -548,5 +551,8 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
return
}
logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
+ // Output the total dollar amount on stdout
+ fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64))
+
return
}
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 36421dc7f..48e7733e1 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -290,6 +290,12 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ // Check that the total amount was printed to stdout
+ c.Check(stdout.String(), check.Matches, "0.01492030\n")
+
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
// Run costanalyzer with 2 container request uuids
exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
commit 8dfbae733b7fa9e9ee579c1fbc1832652cc17485
Author: Ward Vandewege <ward at curii.com>
Date: Sun Dec 6 10:26:39 2020 -0500
17187: costanalyzer: allow specifying multiple comma-separated uuids via
one -uuid argument
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index 75cbdc037..01a9d9fdb 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -44,7 +44,9 @@ func (i *arrayFlags) String() string {
}
func (i *arrayFlags) Set(value string) error {
- *i = append(*i, value)
+ for _, s := range strings.Split(value, ",") {
+ *i = append(*i, s)
+ }
return nil
}
@@ -98,7 +100,7 @@ Options:
}
loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports (required)")
- flags.Var(&uuids, "uuid", "Toplevel `project or container request` uuid. May be specified more than once. (required)")
+ flags.Var(&uuids, "uuid", "object uuid. May be specified more than once. Also accepts a comma separated list of uuids (required)")
flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
err = flags.Parse(args)
if err == flag.ErrHelp {
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 4f0f64dae..36421dc7f 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -285,8 +285,13 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids, as one comma separated -uuid argument
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID + "," + arvadostest.CompletedDiagnosticsContainerRequest2UUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
// Run costanalyzer with 2 container request uuids
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
commit e3c1d52294b815eff176ce8e62ddb0153888bee0
Author: Ward Vandewege <ward at curii.com>
Date: Sat Dec 5 16:37:00 2020 -0500
17187: costanalyzer: add support for collection uuids.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index 4284542b8..75cbdc037 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -62,12 +62,18 @@ Usage:
each container.
When supplied with the uuid of a container request, it will calculate the
- cost of that container request and all its children. When suplied with a
- project uuid or when supplied with multiple container request uuids, it will
- create a CSV report for each supplied uuid, as well as a CSV file with
- aggregate cost accounting for all supplied uuids. The aggregate cost report
- takes container reuse into account: if a container was reused between several
- container requests, its cost will only be counted once.
+ cost of that container request and all its children.
+
+ When supplied with the uuid of a collection, it will see if there is a
+ container_request uuid in the properties of the collection, and if so, it
+ will calculate the cost of that container request and all its children.
+
+ When supplied with a project uuid or when supplied with multiple container
+ request or collection uuids, it will create a CSV report for each supplied
+ uuid, as well as a CSV file with aggregate cost accounting for all supplied
+ uuids. The aggregate cost report takes container reuse into account: if a
+ container was reused between several container requests, its cost will only
+ be counted once.
To get the node costs, the progam queries the Arvados API for current cost
data for each node type used. This means that the reported cost always
@@ -180,8 +186,8 @@ func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.Container
func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
reload = true
- if strings.Contains(uuid, "-j7d0g-") {
- // We do not cache projects, they have no final state
+ if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") {
+ // We do not cache projects or collections, they have no final state
return
}
// See if we have a cached copy of this object
@@ -251,6 +257,8 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str
err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)
} else if strings.Contains(uuid, "-dz642-") {
err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)
+ } else if strings.Contains(uuid, "-4zz18-") {
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil)
} else {
err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err)
return
@@ -309,7 +317,6 @@ func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclien
}
func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
-
cost = make(map[string]float64)
var project arvados.Group
@@ -366,9 +373,24 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
var tmpTotalCost float64
var totalCost float64
+ var crUUID = uuid
+ if strings.Contains(uuid, "-4zz18-") {
+ // This is a collection, find the associated container request (if any)
+ var c arvados.Collection
+ err = loadObject(logger, ac, uuid, uuid, cache, &c)
+ if err != nil {
+ return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err)
+ }
+ value, ok := c.Properties["container_request"]
+ if !ok {
+ return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid)
+ }
+ crUUID = value.(string)
+ }
+
// This is a container request, find the container
var cr arvados.ContainerRequest
- err = loadObject(logger, ac, uuid, uuid, cache, &cr)
+ err = loadObject(logger, ac, crUUID, crUUID, cache, &cr)
if err != nil {
return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
}
@@ -474,12 +496,12 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
for k, v := range cost {
cost[k] = v
}
- } else if strings.Contains(uuid, "-xvhdp-") {
+ } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
// This is a container request
var crCsv map[string]float64
crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
if err != nil {
- err = fmt.Errorf("Error generating container_request CSV for uuid %s: %s", uuid, err.Error())
+ err = fmt.Errorf("Error generating CSV for uuid %s: %s", uuid, err.Error())
exitcode = 2
return
}
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 4fab93bf4..4f0f64dae 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -177,6 +177,48 @@ func (*Suite) TestContainerRequestUUID(c *check.C) {
c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
}
+func (*Suite) TestCollectionUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+
+ // Run costanalyzer with 1 collection uuid, without 'container_request' property
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.FooCollection, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 2)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*does not have a 'container_request' property.*")
+
+ // Update the collection, attach a 'container_request' property
+ ac := arvados.NewClientFromEnv()
+ var coll arvados.Collection
+
+ // Update collection record
+ err := ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+arvadostest.FooCollection, nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "properties": map[string]interface{}{
+ "container_request": arvadostest.CompletedContainerRequestUUID,
+ },
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
+ // Run costanalyzer with 1 collection uuid
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.FooCollection, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+}
+
func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 2 container request uuids
commit 5ea6e759865193c415f86018a07a012731e86a2b
Author: Ward Vandewege <ward at curii.com>
Date: Thu Dec 3 09:13:36 2020 -0500
Upgrade script/create_superuser_token.rb in the api server codebase to
generate v2 tokens.
refs #17022
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/services/api/lib/create_superuser_token.rb b/services/api/lib/create_superuser_token.rb
index 57eac048a..7a18d9705 100755
--- a/services/api/lib/create_superuser_token.rb
+++ b/services/api/lib/create_superuser_token.rb
@@ -54,7 +54,7 @@ module CreateSuperUserToken
end
end
- api_client_auth.api_token
+ "v2/" + api_client_auth.uuid + "/" + api_client_auth.api_token
end
end
end
commit e187e11f44da338014088312c832bfabc3856f0e
Author: Ward Vandewege <ward at curii.com>
Date: Wed Dec 2 20:13:01 2020 -0500
Clarify token details for the arv-user-activity tool.
refs #17022
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/admin/user-activity.html.textile.liquid b/doc/admin/user-activity.html.textile.liquid
index bd8d58833..21bfb7655 100644
--- a/doc/admin/user-activity.html.textile.liquid
+++ b/doc/admin/user-activity.html.textile.liquid
@@ -35,7 +35,7 @@ Note: depends on the "Arvados Python SDK":../sdk/python/sdk-python.html and its
h2. Usage
-Set ARVADOS_API_HOST and ARVADOS_API_TOKEN for an admin user or system root token.
+Set ARVADOS_API_HOST to the api server of the cluster for which the report should be generated. ARVADOS_API_TOKEN needs to be a "v2 token":../admin/scoped-tokens.html for an admin user, or a superuser token (e.g. generated with @script/create_superuser_token.rb@). Please note that in a login cluster federation, the token needs to be issued by the login cluster, but the report should be generated against the API server of the cluster for which it is desired. In other words, ARVADOS_API_HOST would point at the satellite cluster for which the report is desired, but ARVADOS_API_TOKEN would be a token that belongs to a login cluster user.
Run the tool with the option @--days@ giving the number of days to report on. It will request activity logs from the API and generate a summary report on standard output.
commit 33c97ad53a33e005471254cf0b08372c4dabeb4d
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Dec 2 09:30:55 2020 -0500
Update error regexp in test case.
refs #17009
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index b9a6d85ec..562d5296b 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -303,7 +303,7 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
- c.Check(err, check.ErrorMatches, `(mkdir "by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
+ c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
_, err = bucket.GetReader(trial.path)
c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
commit 57ef73c24043dcf5a315ba92eff3a849797b19d0
Author: Javier Bértoli <jbertoli at curii.com>
Date: Wed Dec 2 10:44:27 2020 -0300
fix(docs): broken link
reported on [github](https://github.com/arvados/arvados-formula/issues/3#issuecomment-733019750)
No issue #
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/README.md b/tools/salt-install/README.md
index 3175224d0..10d08b414 100644
--- a/tools/salt-install/README.md
+++ b/tools/salt-install/README.md
@@ -17,4 +17,4 @@ and run it as root.
There's an example `Vagrantfile` also, to install it in a vagrant box if you want
to try it locally.
-For more information, please read https://doc.arvados.org/v2.1/install/install-using-salt.html
+For more information, please read https://doc.arvados.org/main/install/salt-single-host.html
commit 070fcdf489618c3318ada347a876e2e6d21dd0e4
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Nov 25 16:45:53 2020 -0500
17072: Fix for added args on cwltool.docker.get_image
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index d3521099c..7b81bfb44 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -234,7 +234,9 @@ class ArvadosContainer(JobBase):
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
runtimeContext.pull_image,
- runtimeContext.project_uuid)
+ runtimeContext.project_uuid,
+ runtimeContext.force_docker_pull,
+ runtimeContext.tmp_outdir_prefix)
network_req, _ = self.get_requirement("NetworkAccess")
if network_req:
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index a8f56ad1d..3c8208271 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -18,7 +18,8 @@ logger = logging.getLogger('arvados.cwl-runner')
cached_lookups = {}
cached_lookups_lock = threading.Lock()
-def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid):
+def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid,
+ force_pull, tmp_outdir_prefix):
"""Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement:
@@ -48,7 +49,8 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
if not images:
# Fetch Docker image if necessary.
try:
- cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image)
+ cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
+ force_pull, tmp_outdir_prefix)
except OSError as e:
raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index bad8f1e40..42d4b552a 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -443,9 +443,14 @@ def upload_docker(arvrunner, tool):
# TODO: can be supported by containers API, but not jobs API.
raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
"Option 'dockerOutputDirectory' of DockerRequirement not supported.")
- arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid)
+ arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid,
+ arvrunner.runtimeContext.force_docker_pull,
+ arvrunner.runtimeContext.tmp_outdir_prefix)
else:
- arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs:"+__version__}, True, arvrunner.project_uuid)
+ arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs:"+__version__},
+ True, arvrunner.project_uuid,
+ arvrunner.runtimeContext.force_docker_pull,
+ arvrunner.runtimeContext.tmp_outdir_prefix)
elif isinstance(tool, cwltool.workflow.Workflow):
for s in tool.steps:
upload_docker(arvrunner, s.embedded_tool)
@@ -478,7 +483,10 @@ def packed_workflow(arvrunner, tool, merged_map):
if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
if v.get("class") == "DockerRequirement":
- v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True, arvrunner.project_uuid)
+ v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True,
+ arvrunner.project_uuid,
+ arvrunner.runtimeContext.force_docker_pull,
+ arvrunner.runtimeContext.tmp_outdir_prefix)
for l in v:
visit(v[l], cur_id)
if isinstance(v, list):
@@ -583,7 +591,9 @@ def arvados_jobs_image(arvrunner, img):
"""Determine if the right arvados/jobs image version is available. If not, try to pull and upload it."""
try:
- return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
+ return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid,
+ arvrunner.runtimeContext.force_docker_pull,
+ arvrunner.runtimeContext.tmp_outdir_prefix)
except Exception as e:
raise Exception("Docker image %s is not available\n%s" % (img, e) )
commit 320375e3c0071fb3f492323bc3c18e4ca7605fff
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Nov 25 16:13:58 2020 -0500
17072: Increase default submit thread concurrency to 4
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 4bfe27278..3f7f7a972 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -201,7 +201,7 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
help=argparse.SUPPRESS)
parser.add_argument("--thread-count", type=int,
- default=1, help="Number of threads to use for job submit and output collection.")
+ default=4, help="Number of threads to use for job submit and output collection.")
parser.add_argument("--http-timeout", type=int,
default=5*60, dest="http_timeout", help="API request timeout in seconds. Default is 300 seconds (5 minutes).")
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index dc7e32bfe..7aaa27fd6 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -303,7 +303,7 @@ def stubs(func):
'state': 'Committed',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'name': 'submit_wf.cwl',
@@ -413,7 +413,7 @@ class TestSubmit(unittest.TestCase):
expect_container["command"] = [
'arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--disable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -437,7 +437,7 @@ class TestSubmit(unittest.TestCase):
expect_container["command"] = [
'arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["use_existing"] = False
@@ -469,7 +469,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=stop',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -492,7 +492,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256",
"--output-name="+output_name, '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -514,7 +514,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256", "--debug",
"--storage-classes=foo", '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -577,7 +577,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256", '--debug',
'--on-error=continue',
"--intermediate-output-ttl=3600",
@@ -600,7 +600,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
"--trash-intermediate",
@@ -624,7 +624,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256",
"--output-tags="+output_tags, '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -701,7 +701,7 @@ class TestSubmit(unittest.TestCase):
'container_image': '999999999999999999999999999999d3+99',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
@@ -796,7 +796,7 @@ class TestSubmit(unittest.TestCase):
'container_image': "999999999999999999999999999999d3+99",
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
@@ -860,7 +860,7 @@ class TestSubmit(unittest.TestCase):
expect_container["owner_uuid"] = project_uuid
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- "--eval-timeout=20", "--thread-count=1",
+ "--eval-timeout=20", "--thread-count=4",
'--enable-reuse', "--collection-cache-size=256", '--debug',
'--on-error=continue',
'--project-uuid='+project_uuid,
@@ -882,7 +882,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=60.0', '--thread-count=1',
+ '--eval-timeout=60.0', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -903,7 +903,7 @@ class TestSubmit(unittest.TestCase):
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=500",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -995,7 +995,7 @@ class TestSubmit(unittest.TestCase):
}
expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
@@ -1061,7 +1061,7 @@ class TestSubmit(unittest.TestCase):
"--disable-validate",
"--disable-color",
"--eval-timeout=20",
- '--thread-count=1',
+ '--thread-count=4',
"--enable-reuse",
"--collection-cache-size=256",
'--debug',
commit 2fd2dab35b25b09d4923c741bb9fc364299a1a6d
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Nov 25 16:10:57 2020 -0500
17072: Fix imports. Use task_queue from cwltool.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 99d82f339..d3521099c 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -21,8 +21,7 @@ import ruamel.yaml as yaml
from cwltool.errors import WorkflowException
from cwltool.process import UnsupportedRequirement, shortname
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
-from cwltool.utils import aslist
+from cwltool.utils import aslist, adjustFileObjs, adjustDirObjs, visit_class
from cwltool.job import JobBase
import arvados.collection
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 56c6f39e9..6067ae9f4 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -17,7 +17,7 @@ from cwltool.pack import pack
from cwltool.load_tool import fetch_document, resolve_and_validate_document
from cwltool.process import shortname
from cwltool.workflow import Workflow, WorkflowException, WorkflowStep
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
+from cwltool.utils import adjustFileObjs, adjustDirObjs, visit_class
from cwltool.context import LoadingContext
import ruamel.yaml as yaml
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 68141586d..947b630ba 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -37,7 +37,7 @@ from .arvworkflow import ArvadosWorkflow, upload_workflow
from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size
from .perf import Perf
from .pathmapper import NoFollowPathMapper
-from .task_queue import TaskQueue
+from cwltool.task_queue import TaskQueue
from .context import ArvLoadingContext, ArvRuntimeContext
from ._version import __version__
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 5bad29077..e0b2d25bc 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -21,7 +21,9 @@ import arvados.collection
from schema_salad.sourceline import SourceLine
from arvados.errors import ApiError
-from cwltool.pathmapper import PathMapper, MapperEnt, abspath, adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import PathMapper, MapperEnt
+from cwltool.utils import adjustFileObjs, adjustDirObjs
+from cwltool.stdfsaccess import abspath
from cwltool.workflow import WorkflowException
from .http import http_to_keep
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index f06453116..bad8f1e40 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -31,8 +31,7 @@ import cwltool.workflow
from cwltool.process import (scandeps, UnsupportedRequirement, normalizeFilesDirs,
shortname, Process, fill_in_defaults)
from cwltool.load_tool import fetch_document
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
-from cwltool.utils import aslist
+from cwltool.utils import aslist, adjustFileObjs, adjustDirObjs, visit_class
from cwltool.builder import substitute
from cwltool.pack import pack
from cwltool.update import INTERNAL_VERSION
diff --git a/sdk/cwl/arvados_cwl/task_queue.py b/sdk/cwl/arvados_cwl/task_queue.py
deleted file mode 100644
index d75fec6c6..000000000
--- a/sdk/cwl/arvados_cwl/task_queue.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from future import standard_library
-standard_library.install_aliases()
-from builtins import range
-from builtins import object
-
-import queue
-import threading
-import logging
-
-logger = logging.getLogger('arvados.cwl-runner')
-
-class TaskQueue(object):
- def __init__(self, lock, thread_count):
- self.thread_count = thread_count
- self.task_queue = queue.Queue(maxsize=self.thread_count)
- self.task_queue_threads = []
- self.lock = lock
- self.in_flight = 0
- self.error = None
-
- for r in range(0, self.thread_count):
- t = threading.Thread(target=self.task_queue_func)
- self.task_queue_threads.append(t)
- t.start()
-
- def task_queue_func(self):
- while True:
- task = self.task_queue.get()
- if task is None:
- return
- try:
- task()
- except Exception as e:
- logger.exception("Unhandled exception running task")
- self.error = e
-
- with self.lock:
- self.in_flight -= 1
-
- def add(self, task, unlock, check_done):
- if self.thread_count > 1:
- with self.lock:
- self.in_flight += 1
- else:
- task()
- return
-
- while True:
- try:
- unlock.release()
- if check_done.is_set():
- return
- self.task_queue.put(task, block=True, timeout=3)
- return
- except queue.Full:
- pass
- finally:
- unlock.acquire()
-
-
- def drain(self):
- try:
- # Drain queue
- while not self.task_queue.empty():
- self.task_queue.get(True, .1)
- except queue.Empty:
- pass
-
- def join(self):
- for t in self.task_queue_threads:
- self.task_queue.put(None)
- for t in self.task_queue_threads:
- t.join()
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 517ca000b..dc7e32bfe 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -525,7 +525,7 @@ class TestSubmit(unittest.TestCase):
stubs.expect_container_request_uuid + '\n')
self.assertEqual(exited, 0)
- @mock.patch("arvados_cwl.task_queue.TaskQueue")
+ @mock.patch("cwltool.task_queue.TaskQueue")
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
@mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
@stubs
@@ -546,7 +546,7 @@ class TestSubmit(unittest.TestCase):
make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
self.assertEqual(exited, 0)
- @mock.patch("arvados_cwl.task_queue.TaskQueue")
+ @mock.patch("cwltool.task_queue.TaskQueue")
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
@mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
@stubs
diff --git a/sdk/cwl/tests/test_tq.py b/sdk/cwl/tests/test_tq.py
index a09489065..05e5116d7 100644
--- a/sdk/cwl/tests/test_tq.py
+++ b/sdk/cwl/tests/test_tq.py
@@ -11,7 +11,7 @@ import logging
import os
import threading
-from arvados_cwl.task_queue import TaskQueue
+from cwltool.task_queue import TaskQueue
def success_task():
pass
commit 152c62fd0cde25378ee0d2d10c86946dcec8c138
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Nov 25 15:42:08 2020 -0500
17072: Bump cwltool version for unbound 'result' fix
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 334c636dd..a2fba730c 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -39,7 +39,7 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200807132242',
+ 'cwltool==3.0.20201121085451',
'schema-salad==7.0.20200612160654',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
commit b16a207b45d04345a905fe0ac9336eea011026a6
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Nov 30 14:16:56 2020 -0500
17022: Update install docs & fix Marc's email.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/admin/user-activity.html.textile.liquid b/doc/admin/user-activity.html.textile.liquid
index 08d487772..bd8d58833 100644
--- a/doc/admin/user-activity.html.textile.liquid
+++ b/doc/admin/user-activity.html.textile.liquid
@@ -23,15 +23,21 @@ First, configure the "Arvados package repositories":../../install/packages.html
{% include 'install_packages' %}
-h2. Option 2: Install with pip
+h2. Option 2: Install from source
-Run @pip install arvados-user-activity'@ in an appropriate installation environment, such as a @virtualenv at .
+Step 1: Check out the arvados source code
+
+Step 2: Change directory to @arvados/tools/user-activity@
+
+Step 3: Run @pip install .@ in an appropriate installation environment, such as a @virtualenv at .
Note: depends on the "Arvados Python SDK":../sdk/python/sdk-python.html and its associated build prerequisites (e.g. @pycurl@).
h2. Usage
-Set your Arvados environment, then run the tool giving it the number of days to report for. It will query the logs and generate a summary report on standard output.
+Set ARVADOS_API_HOST and ARVADOS_API_TOKEN for an admin user or system root token.
+
+Run the tool with the option @--days@ giving the number of days to report on. It will request activity logs from the API and generate a summary report on standard output.
Example run:
@@ -79,7 +85,7 @@ Peter Amstutz <peter.amstutz at curii.com> (https://workbench.pirca.arvadosapi.com/
2020-11-23 14:53-05:00 to 2020-11-24 11:58-05:00 (21:05) Account activity
2020-11-24 15:06-05:00 to 2020-11-24 16:38-05:00 (01:32) Account activity
-Marc Rubenfield <mrubenfield at gmail.com> (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-v9s9q97pgydh1yf)
+Marc Rubenfield <mrubenfield at curii.com> (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-v9s9q97pgydh1yf)
2020-11-11 12:27-05:00 Untagged pirca-4zz18-xmq257bsla4kdco
2020-11-11 12:27-05:00 Deleted collection "Output of main" (pirca-4zz18-xmq257bsla4kdco)
commit 6ff4379143fe5409eea488d583921b3a68ecb3a2
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 24 18:00:12 2020 -0500
17022: Add example usage of keyset_list_all to code cookbook.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/sdk/python/cookbook.html.textile.liquid b/doc/sdk/python/cookbook.html.textile.liquid
index 82741c3ea..3aa01bbb5 100644
--- a/doc/sdk/python/cookbook.html.textile.liquid
+++ b/doc/sdk/python/cookbook.html.textile.liquid
@@ -257,3 +257,34 @@ for f in files_to_copy:
target.save_new(name=target_name, owner_uuid=target_project)
print("Created collection %s" % target.manifest_locator())
{% endcodeblock %}
+
+h2. Copy files from a collection another collection
+
+{% codeblock as python %}
+import arvados.collection
+
+source_collection = "x1u39-4zz18-krzg64ufvehgitl"
+target_collection = "x1u39-4zz18-67q94einb8ptznm"
+files_to_copy = ["folder1/sample1/sample1_R1.fastq",
+ "folder1/sample2/sample2_R1.fastq"]
+
+source = arvados.collection.CollectionReader(source_collection)
+target = arvados.collection.Collection(target_collection)
+
+for f in files_to_copy:
+ target.copy(f, "", source_collection=source)
+
+target.save()
+{% endcodeblock %}
+
+h2. Listing records with paging
+
+Use the @arvados.util.keyset_list_all@ helper method to iterate over all the records matching an optional filter. This method handles paging internally and returns results incrementally using a Python iterator. The first parameter of the method takes a @list@ method of an Arvados resource (@collections@, @container_requests@, etc).
+
+{% codeblock as python %}
+import arvados.util
+
+api = arvados.api()
+for c in arvados.util.keyset_list_all(api.collections().list, filters=[["name", "like", "%sample123%"]]):
+ print("got collection " + c["uuid"])
+{% endcodeblock %}
commit 17832911c76d4d7f22468b764152fd53f48f0aa2
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 24 17:51:21 2020 -0500
17022: Add test cases for keyset_list_all
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/python/tests/test_util.py b/sdk/python/tests/test_util.py
index 87074dbdf..1c0e437b4 100644
--- a/sdk/python/tests/test_util.py
+++ b/sdk/python/tests/test_util.py
@@ -7,6 +7,7 @@ import subprocess
import unittest
import arvados
+import arvados.util
class MkdirDashPTest(unittest.TestCase):
def setUp(self):
@@ -38,3 +39,139 @@ class RunCommandTestCase(unittest.TestCase):
def test_failure(self):
with self.assertRaises(arvados.errors.CommandFailedError):
arvados.util.run_command(['false'])
+
+class KeysetTestHelper:
+ def __init__(self, expect):
+ self.n = 0
+ self.expect = expect
+
+ def fn(self, **kwargs):
+ if self.expect[self.n][0] != kwargs:
+ raise Exception("Didn't match %s != %s" % (self.expect[self.n][0], kwargs))
+ return self
+
+ def execute(self, num_retries):
+ self.n += 1
+ return self.expect[self.n-1][1]
+
+class KeysetListAllTestCase(unittest.TestCase):
+ def test_empty(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [])
+
+ def test_oneitem(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", "=", "1"], ["uuid", ">", "1"]]},
+ {"items": []}
+ ],[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">", "1"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}])
+
+ def test_onepage2(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}])
+
+ def test_onepage3(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "3", "uuid": "3"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "3"], ["uuid", "!=", "3"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "3", "uuid": "3"}])
+
+
+ def test_twopage(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"]]},
+ {"items": [{"created_at": "3", "uuid": "3"}, {"created_at": "4", "uuid": "4"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "4"], ["uuid", "!=", "4"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"},
+ {"created_at": "2", "uuid": "2"},
+ {"created_at": "3", "uuid": "3"},
+ {"created_at": "4", "uuid": "4"}
+ ])
+
+ def test_repeated_key(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "2", "uuid": "3"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "3"]]},
+ {"items": [{"created_at": "2", "uuid": "2"}, {"created_at": "2", "uuid": "4"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", "=", "2"], ["uuid", ">", "4"]]},
+ {"items": []}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">", "2"]]},
+ {"items": [{"created_at": "3", "uuid": "5"}, {"created_at": "4", "uuid": "6"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "4"], ["uuid", "!=", "6"]]},
+ {"items": []}
+ ],
+ ])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"},
+ {"created_at": "2", "uuid": "2"},
+ {"created_at": "2", "uuid": "3"},
+ {"created_at": "2", "uuid": "4"},
+ {"created_at": "3", "uuid": "5"},
+ {"created_at": "4", "uuid": "6"}
+ ])
+
+ def test_onepage_withfilter(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["foo", ">", "bar"]]},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"], ["foo", ">", "bar"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn, filters=[["foo", ">", "bar"]]))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}])
+
+
+ def test_onepage_desc(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "2", "uuid": "2"}, {"created_at": "1", "uuid": "1"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid asc"], "filters": [["created_at", "<=", "1"], ["uuid", "!=", "1"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn, ascending=False))
+ self.assertEqual(ls, [{"created_at": "2", "uuid": "2"}, {"created_at": "1", "uuid": "1"}])
commit 599507428ab48bfea5dd9a1f25c9ad1f84ab2b0c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 24 17:03:32 2020 -0500
17022: Improve output. Added documentation page.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/build/run-build-packages-python-and-ruby.sh b/build/run-build-packages-python-and-ruby.sh
index f25530760..599fe7cf9 100755
--- a/build/run-build-packages-python-and-ruby.sh
+++ b/build/run-build-packages-python-and-ruby.sh
@@ -203,6 +203,8 @@ if [ $PYTHON -eq 1 ]; then
python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
+ python_wrapper crunchstat_summary "$WORKSPACE/tools/crunchstat-summary"
+ python_wrapper arvados-user-activity "$WORKSPACE/tools/user-activity"
if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
PYTHON_BUILD_FAILURES=$((${#failures[@]} - $GEM_BUILD_FAILURES))
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index ddb21c4cc..42c5f3d09 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -327,7 +327,7 @@ fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "python3"
# The Docker image cleaner
fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
-# The Arvados crunchstat-summary tool
+# The Arvados user activity tool
fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3"
# The cwltest package, which lives out of tree
diff --git a/doc/_config.yml b/doc/_config.yml
index d56a95c1e..75a55b469 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -173,6 +173,7 @@ navbar:
- user/topics/arvados-sync-groups.html.textile.liquid
- admin/scoped-tokens.html.textile.liquid
- admin/token-expiration-policy.html.textile.liquid
+ - admin/user-activity.html.textile.liquid
- Monitoring:
- admin/logging.html.textile.liquid
- admin/metrics.html.textile.liquid
diff --git a/doc/admin/user-activity.html.textile.liquid b/doc/admin/user-activity.html.textile.liquid
new file mode 100644
index 000000000..08d487772
--- /dev/null
+++ b/doc/admin/user-activity.html.textile.liquid
@@ -0,0 +1,95 @@
+---
+layout: default
+navsection: admin
+title: "User activity report"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The @arv-user-activity@ tool generates a summary report of user activity on an Arvados instance based on the audit logs (the @logs@ table).
+
+h2. Installation
+
+h2. Option 1: Install from a distribution package
+
+This installation method is recommended to make the CLI tools available system-wide. It can coexist with the installation method described in option 2, below.
+
+First, configure the "Arvados package repositories":../../install/packages.html
+
+{% assign arvados_component = 'python3-arvados-user-activity' %}
+
+{% include 'install_packages' %}
+
+h2. Option 2: Install with pip
+
+Run @pip install arvados-user-activity'@ in an appropriate installation environment, such as a @virtualenv at .
+
+Note: depends on the "Arvados Python SDK":../sdk/python/sdk-python.html and its associated build prerequisites (e.g. @pycurl@).
+
+h2. Usage
+
+Set your Arvados environment, then run the tool giving it the number of days to report for. It will query the logs and generate a summary report on standard output.
+
+Example run:
+
+<pre>
+$ bin/arv-user-activity --days 14
+User activity on pirca between 2020-11-10 16:42 and 2020-11-24 16:42
+
+Peter Amstutz <peter.amstutz at curii.com> (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-a4qnxq3pcfcgtkz)
+ organization: "Curii"
+ role: "Software Developer"
+
+ 2020-11-10 16:51-05:00 to 2020-11-11 13:51-05:00 (21:00) Account activity
+ 2020-11-13 13:47-05:00 to 2020-11-14 03:32-05:00 (13:45) Account activity
+ 2020-11-14 04:33-05:00 to 2020-11-15 20:33-05:00 (40:00) Account activity
+ 2020-11-15 21:34-05:00 to 2020-11-16 13:34-05:00 (16:00) Account activity
+ 2020-11-16 16:21-05:00 to 2020-11-16 16:28-05:00 (00:07) Account activity
+ 2020-11-17 15:49-05:00 to 2020-11-17 15:49-05:00 (00:00) Account activity
+ 2020-11-17 15:51-05:00 Created project "New project" (pirca-j7d0g-7bxvkyr4khfa1a4)
+ 2020-11-17 15:51-05:00 Updated project "Test run" (pirca-j7d0g-7bxvkyr4khfa1a4)
+ 2020-11-17 15:51-05:00 Ran container "bwa-mem.cwl container" (pirca-xvhdp-xf2w8dkk17jkk5r)
+ 2020-11-17 15:51-05:00 to 2020-11-17 15:51-05:00 (0:00) Account activity
+ 2020-11-17 15:53-05:00 Ran container "WGS processing workflow scattered over samples container" (pirca-xvhdp-u7bm0wdy6lq4r8k)
+ 2020-11-17 15:53-05:00 to 2020-11-17 15:54-05:00 (00:01) Account activity
+ 2020-11-17 15:55-05:00 Created collection "output for pirca-dz642-36ffk81c8zzopxz" (pirca-4zz18-np35gw690ndzzk7)
+ 2020-11-17 15:55-05:00 to 2020-11-17 15:55-05:00 (0:00) Account activity
+ 2020-11-17 15:55-05:00 Created collection "Output of main" (pirca-4zz18-oiiymetwhnnhhwc)
+ 2020-11-17 15:55-05:00 Tagged pirca-4zz18-oiiymetwhnnhhwc
+ 2020-11-17 15:55-05:00 Updated collection "Output of main" (pirca-4zz18-oiiymetwhnnhhwc)
+ 2020-11-17 15:55-05:00 to 2020-11-17 16:04-05:00 (00:09) Account activity
+ 2020-11-17 16:04-05:00 Created collection "Output of main" (pirca-4zz18-f6n9n89e3dhtwvl)
+ 2020-11-17 16:04-05:00 Tagged pirca-4zz18-f6n9n89e3dhtwvl
+ 2020-11-17 16:04-05:00 Updated collection "Output of main" (pirca-4zz18-f6n9n89e3dhtwvl)
+ 2020-11-17 16:04-05:00 to 2020-11-17 17:55-05:00 (01:51) Account activity
+ 2020-11-17 20:09-05:00 to 2020-11-17 20:09-05:00 (00:00) Account activity
+ 2020-11-17 21:35-05:00 to 2020-11-17 21:35-05:00 (00:00) Account activity
+ 2020-11-18 10:09-05:00 to 2020-11-18 11:00-05:00 (00:51) Account activity
+ 2020-11-18 14:37-05:00 Untagged pirca-4zz18-st8yzjan1nhxo1a
+ 2020-11-18 14:37-05:00 Deleted collection "Output of main" (pirca-4zz18-st8yzjan1nhxo1a)
+ 2020-11-18 17:44-05:00 to 2020-11-18 17:44-05:00 (00:00) Account activity
+ 2020-11-19 12:18-05:00 to 2020-11-19 12:19-05:00 (00:01) Account activity
+ 2020-11-19 13:57-05:00 to 2020-11-19 14:21-05:00 (00:24) Account activity
+ 2020-11-20 09:48-05:00 to 2020-11-20 22:51-05:00 (13:03) Account activity
+ 2020-11-20 23:52-05:00 to 2020-11-22 22:32-05:00 (46:40) Account activity
+ 2020-11-22 23:37-05:00 to 2020-11-23 13:52-05:00 (14:15) Account activity
+ 2020-11-23 14:53-05:00 to 2020-11-24 11:58-05:00 (21:05) Account activity
+ 2020-11-24 15:06-05:00 to 2020-11-24 16:38-05:00 (01:32) Account activity
+
+Marc Rubenfield <mrubenfield at gmail.com> (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-v9s9q97pgydh1yf)
+ 2020-11-11 12:27-05:00 Untagged pirca-4zz18-xmq257bsla4kdco
+ 2020-11-11 12:27-05:00 Deleted collection "Output of main" (pirca-4zz18-xmq257bsla4kdco)
+
+Ward Vandewege <ward at curii.com> (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-9z6foyez9ydn2hl)
+ organization: "Curii Corporation, Inc."
+ organization_email: "ward at curii.com"
+ role: "System Administrator"
+ website_url: "https://curii.com"
+
+ 2020-11-19 19:30-05:00 to 2020-11-19 19:46-05:00 (00:16) Account activity
+ 2020-11-20 10:51-05:00 to 2020-11-20 11:26-05:00 (00:35) Account activity
+ 2020-11-24 12:01-05:00 to 2020-11-24 13:01-05:00 (01:00) Account activity
+</pre>
diff --git a/tools/user-activity/arvados_user_activity/main.py b/tools/user-activity/arvados_user_activity/main.py
index d1635c687..959f16d89 100755
--- a/tools/user-activity/arvados_user_activity/main.py
+++ b/tools/user-activity/arvados_user_activity/main.py
@@ -32,9 +32,14 @@ def getowner(arv, uuid, owners):
return getowner(arv, owners[uuid], owners)
-def getusername(arv, uuid):
+def getuserinfo(arv, uuid):
u = arv.users().get(uuid=uuid).execute()
- return "%s %s <%s> (%s)" % (u["first_name"], u["last_name"], u["email"], uuid)
+ prof = "\n".join(" %s: \"%s\"" % (k, v) for k, v in u["prefs"].get("profile", {}).items() if v)
+ if prof:
+ prof = "\n"+prof+"\n"
+ return "%s %s <%s> (%susers/%s)%s" % (u["first_name"], u["last_name"], u["email"],
+ arv.config()["Services"]["Workbench1"]["ExternalURL"],
+ uuid, prof)
def getname(u):
return "\"%s\" (%s)" % (u["name"], u["uuid"])
@@ -49,7 +54,9 @@ def main(arguments=None):
since = datetime.datetime.utcnow() - datetime.timedelta(days=args.days)
- print("Activity since %s\n" % (datetime.datetime.now() - datetime.timedelta(days=args.days)).isoformat())
+ print("User activity on %s between %s and %s\n" % (arv.config()["ClusterID"],
+ (datetime.datetime.now() - datetime.timedelta(days=args.days)).isoformat(sep=" ", timespec="minutes"),
+ datetime.datetime.now().isoformat(sep=" ", timespec="minutes")))
events = arvados.util.keyset_list_all(arv.logs().list, filters=[["created_at", ">=", since.isoformat()]])
@@ -59,7 +66,7 @@ def main(arguments=None):
for e in events:
owner = getowner(arv, e["object_owner_uuid"], owners)
users.setdefault(owner, [])
- event_at = ciso8601.parse_datetime(e["event_at"]).astimezone().isoformat()
+ event_at = ciso8601.parse_datetime(e["event_at"]).astimezone().isoformat(sep=" ", timespec="minutes")
# loguuid = e["uuid"]
loguuid = ""
@@ -87,11 +94,17 @@ def main(arguments=None):
users[owner].append("%s Updated project %s" % (event_at, getname(e["properties"]["new_attributes"])))
elif e["event_type"] in ("create", "update") and e["object_uuid"][6:11] == "gj3su":
+ since_last = None
if len(users[owner]) > 0 and users[owner][-1].endswith("activity"):
sp = users[owner][-1].split(" ")
- users[owner][-1] = "%s to %s Account activity" % (sp[0], event_at)
+ start = sp[0]+" "+sp[1]
+ since_last = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(sp[3]+" "+sp[4])
+ span = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(start)
+
+ if since_last is not None and since_last < datetime.timedelta(minutes=61):
+ users[owner][-1] = "%s to %s (%02d:%02d) Account activity" % (start, event_at, span.days*24 + int(span.seconds/3600), int((span.seconds % 3600)/60))
else:
- users[owner].append("%s Account activity" % (event_at))
+ users[owner].append("%s to %s (0:00) Account activity" % (event_at, event_at))
elif e["event_type"] == "create" and e["object_uuid"][6:11] == "o0j2j":
if e["properties"]["new_attributes"]["link_class"] == "tag":
@@ -130,7 +143,7 @@ def main(arguments=None):
for k,v in users.items():
if k is None or k.endswith("-tpzed-000000000000000"):
continue
- print("%s:" % getusername(arv, k))
+ print(getuserinfo(arv, k))
for ev in v:
print(" %s" % ev)
print("")
diff --git a/tools/user-activity/bin/arv-user-activity b/tools/user-activity/bin/arv-user-activity
new file mode 100755
index 000000000..bc73f84af
--- /dev/null
+++ b/tools/user-activity/bin/arv-user-activity
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+import arvados_user_activity.main
+
+arvados_user_activity.main.main()
commit 486e297e57ab63c5dda916ae24f8e9c29ce30b66
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Nov 18 15:32:21 2020 -0500
17022: Fix packaging
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/user-activity/MANIFEST.in b/tools/user-activity/MANIFEST.in
new file mode 100644
index 000000000..bd1226341
--- /dev/null
+++ b/tools/user-activity/MANIFEST.in
@@ -0,0 +1,6 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+include agpl-3.0.txt
+include arvados_version.py
\ No newline at end of file
diff --git a/tools/user-activity/fpm-info.sh b/tools/user-activity/fpm-info.sh
new file mode 100644
index 000000000..0abc6a08e
--- /dev/null
+++ b/tools/user-activity/fpm-info.sh
@@ -0,0 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+case "$TARGET" in
+ debian* | ubuntu*)
+ fpm_depends+=(libcurl3-gnutls)
+ ;;
+esac
diff --git a/tools/user-activity/setup.py b/tools/user-activity/setup.py
index 6e00a5680..41f8f66a0 100755
--- a/tools/user-activity/setup.py
+++ b/tools/user-activity/setup.py
@@ -31,7 +31,7 @@ setup(name='arvados-user-activity',
('share/doc/arvados_user_activity', ['agpl-3.0.txt']),
],
install_requires=[
- 'arvados-python-client > 2.2.0.dev20201118185221',
+ 'arvados-python-client >= 2.2.0.dev20201118185221',
],
zip_safe=True,
)
commit b379f3bdfa8806947c64e00d81f01e35874c8d45
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Nov 18 15:23:36 2020 -0500
17022: Make this a fully packaged tool.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index 72f814836..8365fecad 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -222,7 +222,8 @@ if test -z "$packages" ; then
python3-arvados-fuse
python3-arvados-python-client
python3-arvados-cwl-runner
- python3-crunchstat-summary"
+ python3-crunchstat-summary
+ python3-arvados-user-activity"
fi
FINAL_EXITCODE=0
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 8d55e2fd9..ddb21c4cc 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -327,6 +327,9 @@ fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "python3"
# The Docker image cleaner
fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
+# The Arvados crunchstat-summary tool
+fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3"
+
# The cwltest package, which lives out of tree
cd "$WORKSPACE"
if [[ -e "$WORKSPACE/cwltest" ]]; then
diff --git a/tools/user-activity/setup.py b/tools/user-activity/setup.py
index 6be50c49c..6e00a5680 100755
--- a/tools/user-activity/setup.py
+++ b/tools/user-activity/setup.py
@@ -16,7 +16,7 @@ README = os.path.join(SETUP_DIR, 'README.rst')
import arvados_version
version = arvados_version.get_version(SETUP_DIR, "arvados_user_activity")
-setup(name='arvados_user_activity',
+setup(name='arvados-user-activity',
version=version,
description='Summarize user activity from Arvados audit logs',
author='Arvados',
@@ -31,7 +31,7 @@ setup(name='arvados_user_activity',
('share/doc/arvados_user_activity', ['agpl-3.0.txt']),
],
install_requires=[
- 'arvados-python-client',
+ 'arvados-python-client > 2.2.0.dev20201118185221',
],
zip_safe=True,
)
commit c924a8fb59cf1455a9ba64726530236d6bc8f141
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Nov 18 15:02:55 2020 -0500
17022: Python packaging for arv-user-activity
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/user-activity/README.rst b/tools/user-activity/README.rst
new file mode 100644
index 000000000..d16ac0823
--- /dev/null
+++ b/tools/user-activity/README.rst
@@ -0,0 +1,5 @@
+.. Copyright (C) The Arvados Authors. All rights reserved.
+..
+.. SPDX-License-Identifier: AGPL-3.0
+
+Summarize user activity from Arvados audit logs
diff --git a/tools/user-activity/agpl-3.0.txt b/tools/user-activity/agpl-3.0.txt
new file mode 100644
index 000000000..dba13ed2d
--- /dev/null
+++ b/tools/user-activity/agpl-3.0.txt
@@ -0,0 +1,661 @@
+ GNU AFFERO GENERAL PUBLIC LICENSE
+ Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+ A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+ The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+ An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU Affero General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Remote Network Interaction; Use with the GNU General Public License.
+
+ Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<http://www.gnu.org/licenses/>.
diff --git a/tools/user-activity/arvados_user_activity/__init__.py b/tools/user-activity/arvados_user_activity/__init__.py
new file mode 100644
index 000000000..e62d75def
--- /dev/null
+++ b/tools/user-activity/arvados_user_activity/__init__.py
@@ -0,0 +1,3 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
diff --git a/tools/user-activity/arv-user-activity.py b/tools/user-activity/arvados_user_activity/main.py
similarity index 96%
rename from tools/user-activity/arv-user-activity.py
rename to tools/user-activity/arvados_user_activity/main.py
index 196133ddd..d1635c687 100755
--- a/tools/user-activity/arv-user-activity.py
+++ b/tools/user-activity/arvados_user_activity/main.py
@@ -13,7 +13,7 @@ import ciso8601
def parse_arguments(arguments):
arg_parser = argparse.ArgumentParser()
- arg_parser.add_argument('--days', type=int)
+ arg_parser.add_argument('--days', type=int, required=True)
args = arg_parser.parse_args(arguments)
return args
@@ -39,7 +39,10 @@ def getusername(arv, uuid):
def getname(u):
return "\"%s\" (%s)" % (u["name"], u["uuid"])
-def main(arguments):
+def main(arguments=None):
+ if arguments is None:
+ arguments = sys.argv[1:]
+
args = parse_arguments(arguments)
arv = arvados.api()
@@ -132,4 +135,5 @@ def main(arguments):
print(" %s" % ev)
print("")
-main(sys.argv[1:])
+if __name__ == "__main__":
+ main()
diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py
new file mode 100644
index 000000000..d8eec3d9e
--- /dev/null
+++ b/tools/user-activity/arvados_version.py
@@ -0,0 +1,58 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+import os
+import re
+import sys
+
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
+
+def choose_version_from():
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ return getver
+
+def git_version_at_commit():
+ curdir = choose_version_from()
+ myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
+ '--format=%H', curdir]).strip()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ return myversion
+
+def save_version(setup_dir, module, v):
+ v = v.replace("~dev", ".dev").replace("~rc", "rc")
+ with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
+
+def read_version(setup_dir, module):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+
+def get_version(setup_dir, module):
+ env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+
+ if env_version:
+ save_version(setup_dir, module, env_version)
+ else:
+ try:
+ save_version(setup_dir, module, git_version_at_commit())
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err), file=sys.stderr)
+ pass
+
+ return read_version(setup_dir, module)
diff --git a/tools/user-activity/setup.py b/tools/user-activity/setup.py
new file mode 100755
index 000000000..6be50c49c
--- /dev/null
+++ b/tools/user-activity/setup.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+from __future__ import absolute_import
+import os
+import sys
+import re
+
+from setuptools import setup, find_packages
+
+SETUP_DIR = os.path.dirname(__file__) or '.'
+README = os.path.join(SETUP_DIR, 'README.rst')
+
+import arvados_version
+version = arvados_version.get_version(SETUP_DIR, "arvados_user_activity")
+
+setup(name='arvados_user_activity',
+ version=version,
+ description='Summarize user activity from Arvados audit logs',
+ author='Arvados',
+ author_email='info at arvados.org',
+ url="https://arvados.org",
+ download_url="https://github.com/arvados/arvados.git",
+ license='GNU Affero General Public License, version 3.0',
+ packages=['arvados_user_activity'],
+ include_package_data=True,
+ entry_points={"console_scripts": ["arv-user-activity=arvados_user_activity.main:main"]},
+ data_files=[
+ ('share/doc/arvados_user_activity', ['agpl-3.0.txt']),
+ ],
+ install_requires=[
+ 'arvados-python-client',
+ ],
+ zip_safe=True,
+)
commit a028f9707926070d8309ceda2b9f88f8625a6ea1
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 17 17:46:42 2020 -0500
17022: Specify time period in days on command line.
Also print timestamps in local time zone.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/user-activity/arv-user-activity.py b/tools/user-activity/arv-user-activity.py
index c8b5365b7..196133ddd 100755
--- a/tools/user-activity/arv-user-activity.py
+++ b/tools/user-activity/arv-user-activity.py
@@ -8,10 +8,12 @@ import sys
import arvados
import arvados.util
+import datetime
+import ciso8601
def parse_arguments(arguments):
arg_parser = argparse.ArgumentParser()
- arg_parser.add_argument('--timespan', type=str)
+ arg_parser.add_argument('--days', type=int)
args = arg_parser.parse_args(arguments)
return args
@@ -32,7 +34,7 @@ def getowner(arv, uuid, owners):
def getusername(arv, uuid):
u = arv.users().get(uuid=uuid).execute()
- return "%s %s (%s)" % (u["first_name"], u["last_name"], uuid)
+ return "%s %s <%s> (%s)" % (u["first_name"], u["last_name"], u["email"], uuid)
def getname(u):
return "\"%s\" (%s)" % (u["name"], u["uuid"])
@@ -42,7 +44,11 @@ def main(arguments):
arv = arvados.api()
- events = arvados.util.keyset_list_all(arv.logs().list, filters=[["created_at", ">=", "2020-10-01T14:51:42-05:00"]])
+ since = datetime.datetime.utcnow() - datetime.timedelta(days=args.days)
+
+ print("Activity since %s\n" % (datetime.datetime.now() - datetime.timedelta(days=args.days)).isoformat())
+
+ events = arvados.util.keyset_list_all(arv.logs().list, filters=[["created_at", ">=", since.isoformat()]])
users = {}
owners = {}
@@ -50,67 +56,73 @@ def main(arguments):
for e in events:
owner = getowner(arv, e["object_owner_uuid"], owners)
users.setdefault(owner, [])
+ event_at = ciso8601.parse_datetime(e["event_at"]).astimezone().isoformat()
+ # loguuid = e["uuid"]
+ loguuid = ""
if e["event_type"] == "create" and e["object_uuid"][6:11] == "tpzed":
users.setdefault(e["object_uuid"], [])
- users[e["object_uuid"]].append("%s User account created" % e["event_at"])
- if e["event_type"] == "update" and e["object_uuid"][6:11] == "tpzed":
+ users[e["object_uuid"]].append("%s User account created" % event_at)
+
+ elif e["event_type"] == "update" and e["object_uuid"][6:11] == "tpzed":
pass
- #users.setdefault(e["object_uuid"], [])
- #users[e["object_uuid"]].append("%s User account created" % e["event_at"])
+
elif e["event_type"] == "create" and e["object_uuid"][6:11] == "xvhdp":
if e["properties"]["new_attributes"]["requesting_container_uuid"] is None:
- users[owner].append("%s Ran container %s %s" % (e["event_at"], getname(e["properties"]["new_attributes"]), e["uuid"]))
+ users[owner].append("%s Ran container %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid))
elif e["event_type"] == "update" and e["object_uuid"][6:11] == "xvhdp":
pass
elif e["event_type"] == "create" and e["object_uuid"][6:11] == "j7d0g":
- users[owner].append("%s Created project %s" % (e["event_at"], getname(e["properties"]["new_attributes"])))
+ users[owner].append("%s Created project %s" % (event_at, getname(e["properties"]["new_attributes"])))
elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "j7d0g":
- users[owner].append("%s Deleted project %s" % (e["event_at"], getname(e["properties"]["old_attributes"])))
+ users[owner].append("%s Deleted project %s" % (event_at, getname(e["properties"]["old_attributes"])))
elif e["event_type"] == "update" and e["object_uuid"][6:11] == "j7d0g":
- users[owner].append("%s Updated project %s" % (e["event_at"], getname(e["properties"]["new_attributes"])))
+ users[owner].append("%s Updated project %s" % (event_at, getname(e["properties"]["new_attributes"])))
elif e["event_type"] in ("create", "update") and e["object_uuid"][6:11] == "gj3su":
if len(users[owner]) > 0 and users[owner][-1].endswith("activity"):
sp = users[owner][-1].split(" ")
- users[owner][-1] = "%s to %s Account activity" % (sp[0], e["event_at"])
+ users[owner][-1] = "%s to %s Account activity" % (sp[0], event_at)
else:
- users[owner].append("%s Account activity" % (e["event_at"]))
+ users[owner].append("%s Account activity" % (event_at))
elif e["event_type"] == "create" and e["object_uuid"][6:11] == "o0j2j":
if e["properties"]["new_attributes"]["link_class"] == "tag":
- users[owner].append("%s Tagged %s" % (e["event_at"], e["properties"]["new_attributes"]["head_uuid"]))
+ users[owner].append("%s Tagged %s" % (event_at, e["properties"]["new_attributes"]["head_uuid"]))
elif e["properties"]["new_attributes"]["link_class"] == "permission":
- users[owner].append("%s Shared %s with %s" % (e["event_at"], e["properties"]["new_attributes"]["tail_uuid"], e["properties"]["new_attributes"]["head_uuid"]))
+ users[owner].append("%s Shared %s with %s" % (event_at, e["properties"]["new_attributes"]["tail_uuid"], e["properties"]["new_attributes"]["head_uuid"]))
else:
- users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], e["uuid"]))
+ users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], loguuid))
elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "o0j2j":
if e["properties"]["old_attributes"]["link_class"] == "tag":
- users[owner].append("%s Untagged %s" % (e["event_at"], e["properties"]["old_attributes"]["head_uuid"]))
+ users[owner].append("%s Untagged %s" % (event_at, e["properties"]["old_attributes"]["head_uuid"]))
elif e["properties"]["old_attributes"]["link_class"] == "permission":
- users[owner].append("%s Unshared %s with %s" % (e["event_at"], e["properties"]["old_attributes"]["tail_uuid"], e["properties"]["old_attributes"]["head_uuid"]))
+ users[owner].append("%s Unshared %s with %s" % (event_at, e["properties"]["old_attributes"]["tail_uuid"], e["properties"]["old_attributes"]["head_uuid"]))
else:
- users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], e["uuid"]))
+ users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], loguuid))
elif e["event_type"] == "create" and e["object_uuid"][6:11] == "4zz18":
if e["properties"]["new_attributes"]["properties"].get("type") in ("log", "output", "intermediate"):
pass
else:
- users[owner].append("%s Created collection %s %s" % (e["event_at"], getname(e["properties"]["new_attributes"]), e["uuid"]))
+ users[owner].append("%s Created collection %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid))
elif e["event_type"] == "update" and e["object_uuid"][6:11] == "4zz18":
- users[owner].append("%s Updated collection %s %s" % (e["event_at"], getname(e["properties"]["new_attributes"]), e["uuid"]))
+ users[owner].append("%s Updated collection %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid))
elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "4zz18":
- users[owner].append("%s Deleted collection %s %s" % (e["event_at"], getname(e["properties"]["old_attributes"]), e["uuid"]))
+ if e["properties"]["old_attributes"]["properties"].get("type") in ("log", "output", "intermediate"):
+ pass
+ else:
+ users[owner].append("%s Deleted collection %s %s" % (event_at, getname(e["properties"]["old_attributes"]), loguuid))
else:
- users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], e["uuid"]))
+ users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], loguuid))
for k,v in users.items():
if k is None or k.endswith("-tpzed-000000000000000"):
commit f563387fb894b0f459d40d4550cf731d930e2bf6
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Nov 13 18:17:54 2020 -0500
17022: Produce user activity report from the audit logs
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 6c9822e9f..2380e48b7 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -388,6 +388,67 @@ def list_all(fn, num_retries=0, **kwargs):
offset = c['offset'] + len(c['items'])
return items
+def keyset_list_all(fn, order_key="created_at", num_retries=0, ascending=True, **kwargs):
+ pagesize = 1000
+ kwargs["limit"] = pagesize
+ kwargs["count"] = 'none'
+ kwargs["order"] = ["%s %s" % (order_key, "asc" if ascending else "desc"), "uuid asc"]
+ other_filters = kwargs.get("filters", [])
+
+ if "select" in kwargs and "uuid" not in kwargs["select"]:
+ kwargs["select"].append("uuid")
+
+ nextpage = []
+ tot = 0
+ expect_full_page = True
+ seen_prevpage = set()
+ seen_thispage = set()
+ lastitem = None
+ prev_page_all_same_order_key = False
+
+ while True:
+ kwargs["filters"] = nextpage+other_filters
+ items = fn(**kwargs).execute(num_retries=num_retries)
+
+ if len(items["items"]) == 0:
+ if prev_page_all_same_order_key:
+ nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
+ prev_page_all_same_order_key = False
+ continue
+ else:
+ return
+
+ seen_prevpage = seen_thispage
+ seen_thispage = set()
+
+ for i in items["items"]:
+ # In cases where there's more than one record with the
+ # same order key, the result could include records we
+ # already saw in the last page. Skip them.
+ if i["uuid"] in seen_prevpage:
+ continue
+ seen_thispage.add(i["uuid"])
+ yield i
+
+ firstitem = items["items"][0]
+ lastitem = items["items"][-1]
+
+ if firstitem[order_key] == lastitem[order_key]:
+ # Got a page where every item has the same order key.
+ # Switch to using uuid for paging.
+ nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">", lastitem["uuid"]]]
+ prev_page_all_same_order_key = True
+ else:
+ # Start from the last order key seen, but skip the last
+ # known uuid to avoid retrieving the same row twice. If
+ # there are multiple rows with the same order key it is
+ # still likely we'll end up retrieving duplicate rows.
+ # That's handled by tracking the "seen" rows for each page
+ # so they can be skipped if they show up on the next page.
+ nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
+ prev_page_all_same_order_key = False
+
+
def ca_certs_path(fallback=httplib2.CA_CERTS):
"""Return the path of the best available CA certs source.
diff --git a/tools/user-activity/arv-user-activity.py b/tools/user-activity/arv-user-activity.py
old mode 100644
new mode 100755
index e5e0f5385..c8b5365b7
--- a/tools/user-activity/arv-user-activity.py
+++ b/tools/user-activity/arv-user-activity.py
@@ -9,99 +9,115 @@ import sys
import arvados
import arvados.util
-def keyset_list_all(fn, order_key="created_at", num_retries=0, ascending=True, **kwargs):
- pagesize = 1000
- kwargs["limit"] = pagesize
- kwargs["count"] = 'none'
- kwargs["order"] = ["%s %s" % (order_key, "asc" if ascending else "desc"), "uuid asc"]
- other_filters = kwargs.get("filters", [])
-
- if "select" in kwargs and "uuid" not in kwargs["select"]:
- kwargs["select"].append("uuid")
-
- nextpage = []
- tot = 0
- expect_full_page = True
- seen_prevpage = set()
- seen_thispage = set()
- lastitem = None
- prev_page_all_same_order_key = False
-
- while True:
- kwargs["filters"] = nextpage+other_filters
- items = fn(**kwargs).execute(num_retries=num_retries)
-
- if len(items["items"]) == 0:
- if prev_page_all_same_order_key:
- nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
- prev_page_all_same_order_key = False
- continue
- else:
- return
-
- seen_prevpage = seen_thispage
- seen_thispage = set()
-
- for i in items["items"]:
- # In cases where there's more than one record with the
- # same order key, the result could include records we
- # already saw in the last page. Skip them.
- if i["uuid"] in seen_prevpage:
- continue
- seen_thispage.add(i["uuid"])
- yield i
-
- firstitem = items["items"][0]
- lastitem = items["items"][-1]
-
- if firstitem[order_key] == lastitem[order_key]:
- # Got a page where every item has the same order key.
- # Switch to using uuid for paging.
- nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">", lastitem["uuid"]]]
- prev_page_all_same_order_key = True
- else:
- # Start from the last order key seen, but skip the last
- # known uuid to avoid retrieving the same row twice. If
- # there are multiple rows with the same order key it is
- # still likely we'll end up retrieving duplicate rows.
- # That's handled by tracking the "seen" rows for each page
- # so they can be skipped if they show up on the next page.
- nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
- prev_page_all_same_order_key = False
-
-
def parse_arguments(arguments):
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--timespan', type=str)
args = arg_parser.parse_args(arguments)
return args
+def getowner(arv, uuid, owners):
+ if uuid is None:
+ return None
+ if uuid[6:11] == "tpzed":
+ return uuid
+
+ if uuid not in owners:
+ try:
+ gp = arv.groups().get(uuid=uuid).execute()
+ owners[uuid] = gp["owner_uuid"]
+ except:
+ owners[uuid] = None
+
+ return getowner(arv, owners[uuid], owners)
+
+def getusername(arv, uuid):
+ u = arv.users().get(uuid=uuid).execute()
+ return "%s %s (%s)" % (u["first_name"], u["last_name"], uuid)
+
+def getname(u):
+ return "\"%s\" (%s)" % (u["name"], u["uuid"])
+
def main(arguments):
args = parse_arguments(arguments)
arv = arvados.api()
- events = keyset_list_all(arv.logs().list, filters=[["created_at", ">=", "2020-11-05T14:51:42-05:00"]])
+ events = arvados.util.keyset_list_all(arv.logs().list, filters=[["created_at", ">=", "2020-10-01T14:51:42-05:00"]])
users = {}
+ owners = {}
for e in events:
+ owner = getowner(arv, e["object_owner_uuid"], owners)
+ users.setdefault(owner, [])
+
if e["event_type"] == "create" and e["object_uuid"][6:11] == "tpzed":
users.setdefault(e["object_uuid"], [])
- users[e["object_uuid"]].append("User was created")
+ users[e["object_uuid"]].append("%s User account created" % e["event_at"])
+ if e["event_type"] == "update" and e["object_uuid"][6:11] == "tpzed":
+ pass
+ #users.setdefault(e["object_uuid"], [])
+ #users[e["object_uuid"]].append("%s User account created" % e["event_at"])
+ elif e["event_type"] == "create" and e["object_uuid"][6:11] == "xvhdp":
+ if e["properties"]["new_attributes"]["requesting_container_uuid"] is None:
+ users[owner].append("%s Ran container %s %s" % (e["event_at"], getname(e["properties"]["new_attributes"]), e["uuid"]))
+
+ elif e["event_type"] == "update" and e["object_uuid"][6:11] == "xvhdp":
+ pass
+
+ elif e["event_type"] == "create" and e["object_uuid"][6:11] == "j7d0g":
+ users[owner].append("%s Created project %s" % (e["event_at"], getname(e["properties"]["new_attributes"])))
+
+ elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "j7d0g":
+ users[owner].append("%s Deleted project %s" % (e["event_at"], getname(e["properties"]["old_attributes"])))
+
+ elif e["event_type"] == "update" and e["object_uuid"][6:11] == "j7d0g":
+ users[owner].append("%s Updated project %s" % (e["event_at"], getname(e["properties"]["new_attributes"])))
+
+ elif e["event_type"] in ("create", "update") and e["object_uuid"][6:11] == "gj3su":
+ if len(users[owner]) > 0 and users[owner][-1].endswith("activity"):
+ sp = users[owner][-1].split(" ")
+ users[owner][-1] = "%s to %s Account activity" % (sp[0], e["event_at"])
+ else:
+ users[owner].append("%s Account activity" % (e["event_at"]))
+
+ elif e["event_type"] == "create" and e["object_uuid"][6:11] == "o0j2j":
+ if e["properties"]["new_attributes"]["link_class"] == "tag":
+ users[owner].append("%s Tagged %s" % (e["event_at"], e["properties"]["new_attributes"]["head_uuid"]))
+ elif e["properties"]["new_attributes"]["link_class"] == "permission":
+ users[owner].append("%s Shared %s with %s" % (e["event_at"], e["properties"]["new_attributes"]["tail_uuid"], e["properties"]["new_attributes"]["head_uuid"]))
+ else:
+ users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], e["uuid"]))
+
+ elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "o0j2j":
+ if e["properties"]["old_attributes"]["link_class"] == "tag":
+ users[owner].append("%s Untagged %s" % (e["event_at"], e["properties"]["old_attributes"]["head_uuid"]))
+ elif e["properties"]["old_attributes"]["link_class"] == "permission":
+ users[owner].append("%s Unshared %s with %s" % (e["event_at"], e["properties"]["old_attributes"]["tail_uuid"], e["properties"]["old_attributes"]["head_uuid"]))
+ else:
+ users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], e["uuid"]))
- if e["event_type"] == "create" and e["object_uuid"][6:11] == "xvhdp":
- users.setdefault(e["object_owner_uuid"], [])
- users[e["object_owner_uuid"]].append("Ran a container")
+ elif e["event_type"] == "create" and e["object_uuid"][6:11] == "4zz18":
+ if e["properties"]["new_attributes"]["properties"].get("type") in ("log", "output", "intermediate"):
+ pass
+ else:
+ users[owner].append("%s Created collection %s %s" % (e["event_at"], getname(e["properties"]["new_attributes"]), e["uuid"]))
+
+ elif e["event_type"] == "update" and e["object_uuid"][6:11] == "4zz18":
+ users[owner].append("%s Updated collection %s %s" % (e["event_at"], getname(e["properties"]["new_attributes"]), e["uuid"]))
- if e["event_type"] == "create" and e["object_uuid"][6:11] == "j7d0g":
- users.setdefault(e["object_owner_uuid"], [])
- users[e["object_owner_uuid"]].append("Created a project")
+ elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "4zz18":
+ users[owner].append("%s Deleted collection %s %s" % (e["event_at"], getname(e["properties"]["old_attributes"]), e["uuid"]))
+
+ else:
+ users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], e["uuid"]))
for k,v in users.items():
- print("%s:" % k)
+ if k is None or k.endswith("-tpzed-000000000000000"):
+ continue
+ print("%s:" % getusername(arv, k))
for ev in v:
print(" %s" % ev)
-
+ print("")
main(sys.argv[1:])
commit 77a773281a967477799d4a586cec59ba40a66ae2
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Nov 9 13:42:12 2020 -0500
17022: Start work on user activity reporting script
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/api/methods.html.textile.liquid b/doc/api/methods.html.textile.liquid
index ae96d0a3b..d6c34f4d3 100644
--- a/doc/api/methods.html.textile.liquid
+++ b/doc/api/methods.html.textile.liquid
@@ -73,9 +73,9 @@ table(table table-bordered table-condensed).
|limit |integer|Maximum number of resources to return. If not provided, server will provide a default limit. Server may also impose a maximum number of records that can be returned in a single request.|query|
|offset |integer|Skip the first 'offset' number of resources that would be returned under the given filter conditions.|query|
|filters |array |"Conditions for selecting resources to return.":#filters|query|
-|order |array |Attributes to use as sort keys to determine the order resources are returned, each optionally followed by @asc@ or @desc@ to indicate ascending or descending order.
+|order |array |Attributes to use as sort keys to determine the order resources are returned, each optionally followed by @asc@ or @desc@ to indicate ascending or descending order. (If not specified, it will be ascending).
Example: @["head_uuid asc","modified_at desc"]@
-Default: @["created_at desc"]@|query|
+Default: @["modified_at desc", "uuid asc"]@|query|
|select |array |Set of attributes to include in the response.
Example: @["head_uuid","tail_uuid"]@
Default: all available attributes. As a special case, collections do not return "manifest_text" unless explicitly selected.|query|
diff --git a/tools/user-activity/arv-user-activity.py b/tools/user-activity/arv-user-activity.py
new file mode 100644
index 000000000..e5e0f5385
--- /dev/null
+++ b/tools/user-activity/arv-user-activity.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+import argparse
+import sys
+
+import arvados
+import arvados.util
+
+def keyset_list_all(fn, order_key="created_at", num_retries=0, ascending=True, **kwargs):
+ pagesize = 1000
+ kwargs["limit"] = pagesize
+ kwargs["count"] = 'none'
+ kwargs["order"] = ["%s %s" % (order_key, "asc" if ascending else "desc"), "uuid asc"]
+ other_filters = kwargs.get("filters", [])
+
+ if "select" in kwargs and "uuid" not in kwargs["select"]:
+ kwargs["select"].append("uuid")
+
+ nextpage = []
+ tot = 0
+ expect_full_page = True
+ seen_prevpage = set()
+ seen_thispage = set()
+ lastitem = None
+ prev_page_all_same_order_key = False
+
+ while True:
+ kwargs["filters"] = nextpage+other_filters
+ items = fn(**kwargs).execute(num_retries=num_retries)
+
+ if len(items["items"]) == 0:
+ if prev_page_all_same_order_key:
+ nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
+ prev_page_all_same_order_key = False
+ continue
+ else:
+ return
+
+ seen_prevpage = seen_thispage
+ seen_thispage = set()
+
+ for i in items["items"]:
+ # In cases where there's more than one record with the
+ # same order key, the result could include records we
+ # already saw in the last page. Skip them.
+ if i["uuid"] in seen_prevpage:
+ continue
+ seen_thispage.add(i["uuid"])
+ yield i
+
+ firstitem = items["items"][0]
+ lastitem = items["items"][-1]
+
+ if firstitem[order_key] == lastitem[order_key]:
+ # Got a page where every item has the same order key.
+ # Switch to using uuid for paging.
+ nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">", lastitem["uuid"]]]
+ prev_page_all_same_order_key = True
+ else:
+ # Start from the last order key seen, but skip the last
+ # known uuid to avoid retrieving the same row twice. If
+ # there are multiple rows with the same order key it is
+ # still likely we'll end up retrieving duplicate rows.
+ # That's handled by tracking the "seen" rows for each page
+ # so they can be skipped if they show up on the next page.
+ nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
+ prev_page_all_same_order_key = False
+
+
+def parse_arguments(arguments):
+ arg_parser = argparse.ArgumentParser()
+ arg_parser.add_argument('--timespan', type=str)
+ args = arg_parser.parse_args(arguments)
+ return args
+
+def main(arguments):
+ args = parse_arguments(arguments)
+
+ arv = arvados.api()
+
+ events = keyset_list_all(arv.logs().list, filters=[["created_at", ">=", "2020-11-05T14:51:42-05:00"]])
+
+ users = {}
+
+ for e in events:
+ if e["event_type"] == "create" and e["object_uuid"][6:11] == "tpzed":
+ users.setdefault(e["object_uuid"], [])
+ users[e["object_uuid"]].append("User was created")
+
+ if e["event_type"] == "create" and e["object_uuid"][6:11] == "xvhdp":
+ users.setdefault(e["object_owner_uuid"], [])
+ users[e["object_owner_uuid"]].append("Ran a container")
+
+ if e["event_type"] == "create" and e["object_uuid"][6:11] == "j7d0g":
+ users.setdefault(e["object_owner_uuid"], [])
+ users[e["object_owner_uuid"]].append("Created a project")
+
+ for k,v in users.items():
+ print("%s:" % k)
+ for ev in v:
+ print(" %s" % ev)
+
+
+main(sys.argv[1:])
commit 70f4b1a134dd64c18498c880d218bfc8187349ed
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 30 11:55:41 2020 -0500
Now that our k8s helm charts retain state, update the documentation
accordingly.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid b/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid
index 0801b7d4e..06280b467 100644
--- a/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid
+++ b/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid
@@ -11,10 +11,6 @@ SPDX-License-Identifier: CC-BY-SA-3.0
This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Google Kubernetes Engine@ (GKE).
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
h2. Prerequisites
h3. Install tooling
@@ -142,10 +138,6 @@ $ helm upgrade arvados .
h2. Shut down
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
<pre>
$ helm del arvados
</pre>
diff --git a/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid b/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid
index 86aaf08f9..9ecb2c895 100644
--- a/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid
+++ b/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid
@@ -11,10 +11,6 @@ SPDX-License-Identifier: CC-BY-SA-3.0
This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Minikube at .
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
h2. Prerequisites
h3. Install tooling
@@ -128,7 +124,7 @@ $ helm upgrade arvados .
h2. Shut down
{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
+This Helm chart uses Kubernetes <i>persistent volumes</i> for the Postgresql and Keepstore data volumes. These volumes will be retained after you delete the Arvados helm chart with the command below. Because those volumes are stored in the local Minikube Kubernetes cluster, if you delete that cluster (e.g. with <i>minikube delete</i>) the Kubernetes persistent volumes will also be deleted.
{% include 'notebox_end' %}
<pre>
diff --git a/doc/install/arvados-on-kubernetes.html.textile.liquid b/doc/install/arvados-on-kubernetes.html.textile.liquid
index ff52aa171..9169b7810 100644
--- a/doc/install/arvados-on-kubernetes.html.textile.liquid
+++ b/doc/install/arvados-on-kubernetes.html.textile.liquid
@@ -11,10 +11,6 @@ SPDX-License-Identifier: CC-BY-SA-3.0
Arvados on Kubernetes is implemented as a @Helm 3@ chart.
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
h2(#overview). Overview
This Helm chart provides a basic, small Arvados cluster.
commit 06227b8b8a1d5c3488a35342444dbb2aa59684e5
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 24 13:40:21 2020 -0500
16774: text/plain response uses crlf. Tests check error codes.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index f2e9ba5aa..ab1bc080b 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -62,8 +62,8 @@ func parseCollectionIDFromDNSName(s string) string {
var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
-var notFoundMessage = "404 Not found\n\nThe requested path was not found, or you do not have permission to access it."
-var unauthorizedMessage = "401 Unauthorized\n\nA valid Arvados token must be provided to access this resource."
+var notFoundMessage = "404 Not found\r\n\r\nThe requested path was not found, or you do not have permission to access it.\r"
+var unauthorizedMessage = "401 Unauthorized\r\n\r\nA valid Arvados token must be provided to access this resource.\r"
// parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a
// PDH (even if it is a PDH with "+" replaced by " " or "-");
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index afdca4718..b9a6d85ec 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -178,6 +178,8 @@ func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix
// GetObject
rdr, err = bucket.GetReader(prefix + "missingfile")
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
// HeadObject
@@ -240,6 +242,8 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket,
objname := prefix + trial.path
_, err := bucket.GetReader(objname)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
buf := make([]byte, trial.size)
@@ -289,15 +293,21 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
c.Logf("=== %v", trial)
_, err := bucket.GetReader(trial.path)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
buf := make([]byte, trial.size)
rand.Read(buf)
err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
+ c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
c.Check(err, check.ErrorMatches, `(mkdir "by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
_, err = bucket.GetReader(trial.path)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
}
}
@@ -406,6 +416,8 @@ func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket,
if objname != "" && objname != "/" {
_, err = bucket.GetReader(objname)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
}
}()
commit 41cdc93d9aa3b119322bf1e666c96d235c36e43c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Nov 23 15:04:48 2020 -0500
16774: Fix tests. Use encoder for xml error response.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index f6f3de887..8e2e05c76 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -122,7 +122,7 @@ func (s *IntegrationSuite) TestVhost404(c *check.C) {
}
s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
- c.Check(resp.Body.String(), check.Equals, "")
+ c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
}
}
@@ -250,7 +250,11 @@ func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authori
// depending on the authz method.
c.Check(code, check.Equals, failCode)
}
- c.Check(body, check.Equals, "")
+ if code == 404 {
+ c.Check(body, check.Equals, notFoundMessage+"\n")
+ } else {
+ c.Check(body, check.Equals, unauthorizedMessage+"\n")
+ }
}
}
}
@@ -307,7 +311,7 @@ func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
"",
"",
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
@@ -321,7 +325,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C)
"",
"",
http.StatusUnauthorized,
- "",
+ unauthorizedMessage+"\n",
)
}
@@ -439,7 +443,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C)
"application/x-www-form-urlencoded",
url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
@@ -463,7 +467,7 @@ func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
"",
"",
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 08eb7954a..7fb90789a 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -193,13 +193,19 @@ func s3ErrorResponse(w http.ResponseWriter, s3code string, message string, resou
w.Header().Set("Content-Type", "application/xml")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.WriteHeader(code)
- fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
-<Error>
- <Code>%v</Code>
- <Message>%v</Message>
- <Resource>%v</Resource>
- <RequestId></RequestId>
-</Error>`, code, message, resource)
+ var errstruct struct {
+ Code string
+ Message string
+ Resource string
+ RequestId string
+ }
+ errstruct.Code = s3code
+ errstruct.Message = message
+ errstruct.Resource = resource
+ errstruct.RequestId = ""
+ enc := xml.NewEncoder(w)
+ fmt.Fprint(w, xml.Header)
+ enc.EncodeElement(errstruct, xml.StartElement{Name: xml.Name{Local: "Error"}})
}
var NoSuchKey = "NoSuchKey"
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index bff197ded..afdca4718 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -178,7 +178,7 @@ func (s *IntegrationSuite) testS3GetObject(c *check.C, bucket *s3.Bucket, prefix
// GetObject
rdr, err = bucket.GetReader(prefix + "missingfile")
- c.Check(err, check.ErrorMatches, `404 Not Found`)
+ c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
// HeadObject
exists, err := bucket.Exists(prefix + "missingfile")
@@ -240,7 +240,7 @@ func (s *IntegrationSuite) testS3PutObjectSuccess(c *check.C, bucket *s3.Bucket,
objname := prefix + trial.path
_, err := bucket.GetReader(objname)
- c.Assert(err, check.ErrorMatches, `404 Not Found`)
+ c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
buf := make([]byte, trial.size)
rand.Read(buf)
@@ -289,16 +289,16 @@ func (s *IntegrationSuite) TestS3ProjectPutObjectNotSupported(c *check.C) {
c.Logf("=== %v", trial)
_, err := bucket.GetReader(trial.path)
- c.Assert(err, check.ErrorMatches, `404 Not Found`)
+ c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
buf := make([]byte, trial.size)
rand.Read(buf)
err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
- c.Check(err, check.ErrorMatches, `400 Bad Request`)
+ c.Check(err, check.ErrorMatches, `(mkdir "by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
_, err = bucket.GetReader(trial.path)
- c.Assert(err, check.ErrorMatches, `404 Not Found`)
+ c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
}
}
@@ -400,13 +400,13 @@ func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket,
rand.Read(buf)
err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
- if !c.Check(err, check.ErrorMatches, `400 Bad.*`, check.Commentf("PUT %q should fail", objname)) {
+ if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
return
}
if objname != "" && objname != "/" {
_, err = bucket.GetReader(objname)
- c.Check(err, check.ErrorMatches, `404 Not Found`, check.Commentf("GET %q should return 404", objname))
+ c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
}
}()
}
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 43817b51f..0a1c7d1b3 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -43,17 +43,17 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
} {
hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, "")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
if token != "" {
hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, "")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
}
hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, "")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
}
}
@@ -86,7 +86,7 @@ func (s *IntegrationSuite) Test404(c *check.C) {
hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
if len(body) > 0 {
- c.Check(body, check.Equals, "404 page not found\n")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
}
}
}
commit 20b80f0f28588b3c7961f5ecdfc8af2659affabc
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Nov 20 18:03:30 2020 -0500
16774: Keep-web errors include messages
Errors on the regular keep-web side return plain text responses.
Errors on the S3 side return XML error responses that S3 clients
expect.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 963948cc6..f2e9ba5aa 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -62,6 +62,9 @@ func parseCollectionIDFromDNSName(s string) string {
var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
+var notFoundMessage = "404 Not found\n\nThe requested path was not found, or you do not have permission to access it."
+var unauthorizedMessage = "401 Unauthorized\n\nA valid Arvados token must be provided to access this resource."
+
// parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a
// PDH (even if it is a PDH with "+" replaced by " " or "-");
// otherwise "".
@@ -279,7 +282,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
if collectionID == "" && !useSiteFS {
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
return
}
@@ -388,14 +391,14 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// for additional credentials would just be
// confusing), or we don't even accept
// credentials at this path.
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
return
}
for _, t := range reqTokens {
if tokenResult[t] == 404 {
// The client provided valid token(s), but the
// collection was not found.
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
return
}
}
@@ -409,7 +412,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// data that has been deleted. Allow a referrer to
// provide this context somehow?
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
- w.WriteHeader(http.StatusUnauthorized)
+ http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
return
}
@@ -479,7 +482,7 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
openPath := "/" + strings.Join(targetPath, "/")
if f, err := fs.Open(openPath); os.IsNotExist(err) {
// Requested non-existent path
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
} else if err != nil {
// Some other (unexpected) error
http.Error(w, "open: "+err.Error(), http.StatusInternalServerError)
@@ -533,7 +536,7 @@ func (h *handler) getClients(reqID, token string) (arv *arvadosclient.ArvadosCli
func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK, attachment bool) {
if len(tokens) == 0 {
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
- http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
+ http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
return
}
if writeMethod[r.Method] {
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 373fd9a25..08eb7954a 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -189,6 +189,27 @@ func (h *handler) checks3signature(r *http.Request) (string, error) {
return aca.TokenV2(), nil
}
+func s3ErrorResponse(w http.ResponseWriter, s3code string, message string, resource string, code int) {
+ w.Header().Set("Content-Type", "application/xml")
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ w.WriteHeader(code)
+ fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
+<Error>
+ <Code>%v</Code>
+ <Message>%v</Message>
+ <Resource>%v</Resource>
+ <RequestId></RequestId>
+</Error>`, code, message, resource)
+}
+
+var NoSuchKey = "NoSuchKey"
+var NoSuchBucket = "NoSuchBucket"
+var InvalidArgument = "InvalidArgument"
+var InternalError = "InternalError"
+var UnauthorizedAccess = "UnauthorizedAccess"
+var InvalidRequest = "InvalidRequest"
+var SignatureDoesNotMatch = "SignatureDoesNotMatch"
+
// serveS3 handles r and returns true if r is a request from an S3
// client, otherwise it returns false.
func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
@@ -196,14 +217,14 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "AWS ") {
split := strings.SplitN(auth[4:], ":", 2)
if len(split) < 2 {
- http.Error(w, "malformed Authorization header", http.StatusUnauthorized)
+ s3ErrorResponse(w, InvalidRequest, "malformed Authorization header", r.URL.Path, http.StatusUnauthorized)
return true
}
token = unescapeKey(split[0])
} else if strings.HasPrefix(auth, s3SignAlgorithm+" ") {
t, err := h.checks3signature(r)
if err != nil {
- http.Error(w, "signature verification failed: "+err.Error(), http.StatusForbidden)
+ s3ErrorResponse(w, SignatureDoesNotMatch, "signature verification failed: "+err.Error(), r.URL.Path, http.StatusForbidden)
return true
}
token = t
@@ -213,7 +234,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
_, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token)
if err != nil {
- http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, "Pool failed: "+h.clientPool.Err().Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
defer release()
@@ -251,9 +272,9 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
if err == nil && fi.IsDir() {
w.WriteHeader(http.StatusOK)
} else if os.IsNotExist(err) {
- w.WriteHeader(http.StatusNotFound)
+ s3ErrorResponse(w, NoSuchBucket, "The specified bucket does not exist.", r.URL.Path, http.StatusNotFound)
} else {
- http.Error(w, err.Error(), http.StatusBadGateway)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
}
return true
}
@@ -265,7 +286,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
if os.IsNotExist(err) ||
(err != nil && err.Error() == "not a directory") ||
(fi != nil && fi.IsDir()) {
- http.Error(w, "not found", http.StatusNotFound)
+ s3ErrorResponse(w, NoSuchKey, "The specified key does not exist.", r.URL.Path, http.StatusNotFound)
return true
}
// shallow copy r, and change URL path
@@ -275,24 +296,24 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
return true
case r.Method == http.MethodPut:
if !objectNameGiven {
- http.Error(w, "missing object name in PUT request", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "Missing object name in PUT request.", r.URL.Path, http.StatusBadRequest)
return true
}
var objectIsDir bool
if strings.HasSuffix(fspath, "/") {
if !h.Config.cluster.Collections.S3FolderObjects {
- http.Error(w, "invalid object name: trailing slash", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "invalid object name: trailing slash", r.URL.Path, http.StatusBadRequest)
return true
}
n, err := r.Body.Read(make([]byte, 1))
if err != nil && err != io.EOF {
- http.Error(w, fmt.Sprintf("error reading request body: %s", err), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, fmt.Sprintf("error reading request body: %s", err), r.URL.Path, http.StatusInternalServerError)
return true
} else if n > 0 {
- http.Error(w, "cannot create object with trailing '/' char unless content is empty", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "cannot create object with trailing '/' char unless content is empty", r.URL.Path, http.StatusBadRequest)
return true
} else if strings.SplitN(r.Header.Get("Content-Type"), ";", 2)[0] != "application/x-directory" {
- http.Error(w, "cannot create object with trailing '/' char unless Content-Type is 'application/x-directory'", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "cannot create object with trailing '/' char unless Content-Type is 'application/x-directory'", r.URL.Path, http.StatusBadRequest)
return true
}
// Given PUT "foo/bar/", we'll use "foo/bar/."
@@ -304,12 +325,12 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
fi, err := fs.Stat(fspath)
if err != nil && err.Error() == "not a directory" {
// requested foo/bar, but foo is a file
- http.Error(w, "object name conflicts with existing object", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "object name conflicts with existing object", r.URL.Path, http.StatusBadRequest)
return true
}
if strings.HasSuffix(r.URL.Path, "/") && err == nil && !fi.IsDir() {
// requested foo/bar/, but foo/bar is a file
- http.Error(w, "object name conflicts with existing object", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "object name conflicts with existing object", r.URL.Path, http.StatusBadRequest)
return true
}
// create missing parent/intermediate directories, if any
@@ -318,7 +339,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
dir := fspath[:i]
if strings.HasSuffix(dir, "/") {
err = errors.New("invalid object name (consecutive '/' chars)")
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
}
err = fs.Mkdir(dir, 0755)
@@ -326,11 +347,11 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
// Cannot create a directory
// here.
err = fmt.Errorf("mkdir %q failed: %w", dir, err)
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
} else if err != nil && !os.IsExist(err) {
err = fmt.Errorf("mkdir %q failed: %w", dir, err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
}
@@ -342,34 +363,34 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
}
if err != nil {
err = fmt.Errorf("open %q failed: %w", r.URL.Path, err)
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
}
defer f.Close()
_, err = io.Copy(f, r.Body)
if err != nil {
err = fmt.Errorf("write to %q failed: %w", r.URL.Path, err)
- http.Error(w, err.Error(), http.StatusBadGateway)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
return true
}
err = f.Close()
if err != nil {
err = fmt.Errorf("write to %q failed: close: %w", r.URL.Path, err)
- http.Error(w, err.Error(), http.StatusBadGateway)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
return true
}
}
err = fs.Sync()
if err != nil {
err = fmt.Errorf("sync failed: %w", err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
w.WriteHeader(http.StatusOK)
return true
case r.Method == http.MethodDelete:
if !objectNameGiven || r.URL.Path == "/" {
- http.Error(w, "missing object name in DELETE request", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "missing object name in DELETE request", r.URL.Path, http.StatusBadRequest)
return true
}
if strings.HasSuffix(fspath, "/") {
@@ -379,7 +400,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
w.WriteHeader(http.StatusNoContent)
return true
} else if err != nil {
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
} else if !fi.IsDir() {
// if "foo" exists and is a file, then
@@ -403,19 +424,20 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
}
if err != nil {
err = fmt.Errorf("rm failed: %w", err)
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
}
err = fs.Sync()
if err != nil {
err = fmt.Errorf("sync failed: %w", err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
w.WriteHeader(http.StatusNoContent)
return true
default:
- http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+ s3ErrorResponse(w, InvalidRequest, "method not allowed", r.URL.Path, http.StatusMethodNotAllowed)
+
return true
}
}
commit bf53079ba22cfa04b3fbac5b8dcf4467844601f8
Author: Ward Vandewege <ward at curii.com>
Date: Sun Nov 29 20:51:27 2020 -0500
Fix more golint warnings - update crunchrun tests.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go
index 55cc6ee56..02ad1d0e2 100644
--- a/lib/crunchrun/crunchrun_test.go
+++ b/lib/crunchrun/crunchrun_test.go
@@ -1506,7 +1506,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
err := cr.SetupMounts()
c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
+ c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
@@ -1523,7 +1523,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
err := cr.SetupMounts()
c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`)
+ c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
@@ -1654,7 +1654,7 @@ func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
}`, func(t *TestDockerClient) {})
c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'tmp' for stdout"), Equals, true)
+ c.Check(strings.Contains(err.Error(), "unsupported mount kind 'tmp' for stdout"), Equals, true)
}
func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
@@ -1665,7 +1665,7 @@ func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
}`, func(t *TestDockerClient) {})
c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'collection' for stdout"), Equals, true)
+ c.Check(strings.Contains(err.Error(), "unsupported mount kind 'collection' for stdout"), Equals, true)
}
func (s *TestSuite) TestFullRunWithAPI(c *C) {
commit dc5611c404d0d32994b07037309935ec717c12c2
Author: Ward Vandewege <ward at curii.com>
Date: Sat Nov 28 21:26:28 2020 -0500
Fix more golint warnings.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/boot/postgresql.go b/lib/boot/postgresql.go
index 34ccf04a8..7661c6b58 100644
--- a/lib/boot/postgresql.go
+++ b/lib/boot/postgresql.go
@@ -61,7 +61,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
if err != nil {
return fmt.Errorf("user.Lookup(\"postgres\"): %s", err)
}
- postgresUid, err := strconv.Atoi(postgresUser.Uid)
+ postgresUID, err := strconv.Atoi(postgresUser.Uid)
if err != nil {
return fmt.Errorf("user.Lookup(\"postgres\"): non-numeric uid?: %q", postgresUser.Uid)
}
@@ -77,7 +77,7 @@ func (runPostgreSQL) Run(ctx context.Context, fail func(error), super *Superviso
if err != nil {
return err
}
- err = os.Chown(datadir, postgresUid, 0)
+ err = os.Chown(datadir, postgresUID, 0)
if err != nil {
return err
}
diff --git a/lib/controller/fed_containers.go b/lib/controller/fed_containers.go
index 028f4f597..f588f71af 100644
--- a/lib/controller/fed_containers.go
+++ b/lib/controller/fed_containers.go
@@ -65,14 +65,14 @@ func remoteContainerRequestCreate(
crString, ok := request["container_request"].(string)
if ok {
- var crJson map[string]interface{}
- err := json.Unmarshal([]byte(crString), &crJson)
+ var crJSON map[string]interface{}
+ err := json.Unmarshal([]byte(crString), &crJSON)
if err != nil {
httpserver.Error(w, err.Error(), http.StatusBadRequest)
return true
}
- request["container_request"] = crJson
+ request["container_request"] = crJSON
}
containerRequest, ok := request["container_request"].(map[string]interface{})
diff --git a/lib/crunchrun/background.go b/lib/crunchrun/background.go
index da5361079..4bb249380 100644
--- a/lib/crunchrun/background.go
+++ b/lib/crunchrun/background.go
@@ -132,7 +132,7 @@ func kill(uuid string, signal syscall.Signal, stdout, stderr io.Writer) error {
var pi procinfo
err = json.NewDecoder(f).Decode(&pi)
if err != nil {
- return fmt.Errorf("decode %s: %s\n", path, err)
+ return fmt.Errorf("decode %s: %s", path, err)
}
if pi.UUID != uuid || pi.PID == 0 {
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index c125b27a5..3a4f3a102 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -455,11 +455,11 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
}
for bind := range runner.SecretMounts {
if _, ok := runner.Container.Mounts[bind]; ok {
- return fmt.Errorf("Secret mount %q conflicts with regular mount", bind)
+ return fmt.Errorf("secret mount %q conflicts with regular mount", bind)
}
if runner.SecretMounts[bind].Kind != "json" &&
runner.SecretMounts[bind].Kind != "text" {
- return fmt.Errorf("Secret mount %q type is %q but only 'json' and 'text' are permitted.",
+ return fmt.Errorf("secret mount %q type is %q but only 'json' and 'text' are permitted",
bind, runner.SecretMounts[bind].Kind)
}
binds = append(binds, bind)
@@ -474,7 +474,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
if bind == "stdout" || bind == "stderr" {
// Is it a "file" mount kind?
if mnt.Kind != "file" {
- return fmt.Errorf("Unsupported mount kind '%s' for %s. Only 'file' is supported.", mnt.Kind, bind)
+ return fmt.Errorf("unsupported mount kind '%s' for %s: only 'file' is supported", mnt.Kind, bind)
}
// Does path start with OutputPath?
@@ -490,7 +490,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
if bind == "stdin" {
// Is it a "collection" mount kind?
if mnt.Kind != "collection" && mnt.Kind != "json" {
- return fmt.Errorf("Unsupported mount kind '%s' for stdin. Only 'collection' or 'json' are supported.", mnt.Kind)
+ return fmt.Errorf("unsupported mount kind '%s' for stdin: only 'collection' and 'json' are supported", mnt.Kind)
}
}
@@ -500,7 +500,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
if strings.HasPrefix(bind, runner.Container.OutputPath+"/") && bind != runner.Container.OutputPath+"/" {
if mnt.Kind != "collection" && mnt.Kind != "text" && mnt.Kind != "json" {
- return fmt.Errorf("Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
+ return fmt.Errorf("only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
}
}
@@ -508,17 +508,17 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
case mnt.Kind == "collection" && bind != "stdin":
var src string
if mnt.UUID != "" && mnt.PortableDataHash != "" {
- return fmt.Errorf("Cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
+ return fmt.Errorf("cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
}
if mnt.UUID != "" {
if mnt.Writable {
- return fmt.Errorf("Writing to existing collections currently not permitted.")
+ return fmt.Errorf("writing to existing collections currently not permitted")
}
pdhOnly = false
src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.UUID)
} else if mnt.PortableDataHash != "" {
if mnt.Writable && !strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
- return fmt.Errorf("Can never write to a collection specified by portable data hash")
+ return fmt.Errorf("can never write to a collection specified by portable data hash")
}
idx := strings.Index(mnt.PortableDataHash, "/")
if idx > 0 {
@@ -559,15 +559,15 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
var tmpdir string
tmpdir, err = runner.MkTempDir(runner.parentTemp, "tmp")
if err != nil {
- return fmt.Errorf("While creating mount temp dir: %v", err)
+ return fmt.Errorf("while creating mount temp dir: %v", err)
}
st, staterr := os.Stat(tmpdir)
if staterr != nil {
- return fmt.Errorf("While Stat on temp dir: %v", staterr)
+ return fmt.Errorf("while Stat on temp dir: %v", staterr)
}
err = os.Chmod(tmpdir, st.Mode()|os.ModeSetgid|0777)
if staterr != nil {
- return fmt.Errorf("While Chmod temp dir: %v", err)
+ return fmt.Errorf("while Chmod temp dir: %v", err)
}
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", tmpdir, bind))
if bind == runner.Container.OutputPath {
@@ -618,7 +618,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
}
if runner.HostOutputDir == "" {
- return fmt.Errorf("Output path does not correspond to a writable mount point")
+ return fmt.Errorf("output path does not correspond to a writable mount point")
}
if wantAPI := runner.Container.RuntimeConstraints.API; needCertMount && wantAPI != nil && *wantAPI {
@@ -640,20 +640,20 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
runner.ArvMount, err = runner.RunArvMount(arvMountCmd, token)
if err != nil {
- return fmt.Errorf("While trying to start arv-mount: %v", err)
+ return fmt.Errorf("while trying to start arv-mount: %v", err)
}
for _, p := range collectionPaths {
_, err = os.Stat(p)
if err != nil {
- return fmt.Errorf("While checking that input files exist: %v", err)
+ return fmt.Errorf("while checking that input files exist: %v", err)
}
}
for _, cp := range copyFiles {
st, err := os.Stat(cp.src)
if err != nil {
- return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err)
+ return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
}
if st.IsDir() {
err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error {
@@ -674,7 +674,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
}
return os.Chmod(target, walkinfo.Mode()|os.ModeSetgid|0777)
} else {
- return fmt.Errorf("Source %q is not a regular file or directory", cp.src)
+ return fmt.Errorf("source %q is not a regular file or directory", cp.src)
}
})
} else if st.Mode().IsRegular() {
@@ -684,7 +684,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
}
}
if err != nil {
- return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err)
+ return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
}
}
commit 1abc152750e2780c8c388f28825ba7594ee55722
Author: Ward Vandewege <ward at curii.com>
Date: Fri Nov 27 08:28:41 2020 -0500
Fix more golint warnings.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 2b2d72f4f..b04757ac3 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -25,6 +25,7 @@ import (
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
)
diff --git a/lib/ctrlctx/db.go b/lib/ctrlctx/db.go
index 127be489d..36d79d3d2 100644
--- a/lib/ctrlctx/db.go
+++ b/lib/ctrlctx/db.go
@@ -12,6 +12,7 @@ import (
"git.arvados.org/arvados.git/lib/controller/api"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
)
diff --git a/lib/mount/command.go b/lib/mount/command.go
index 86a9085bd..e92af2407 100644
--- a/lib/mount/command.go
+++ b/lib/mount/command.go
@@ -9,6 +9,7 @@ import (
"io"
"log"
"net/http"
+ // pprof is only imported to register its HTTP handlers
_ "net/http/pprof"
"os"
diff --git a/sdk/go/arvadostest/db.go b/sdk/go/arvadostest/db.go
index 41ecfacc4..c20f61db2 100644
--- a/sdk/go/arvadostest/db.go
+++ b/sdk/go/arvadostest/db.go
@@ -10,6 +10,7 @@ import (
"git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
"gopkg.in/check.v1"
)
commit 3b9c8adfe9f4cf8966d70fad927affaae0a11802
Author: Ward Vandewege <ward at curii.com>
Date: Thu Nov 26 17:17:10 2020 -0500
Fix more golint warnings.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/sdk/go/keepclient/hashcheck.go b/sdk/go/keepclient/hashcheck.go
index 9295c14cc..0966e072e 100644
--- a/sdk/go/keepclient/hashcheck.go
+++ b/sdk/go/keepclient/hashcheck.go
@@ -29,36 +29,36 @@ type HashCheckingReader struct {
// Reads from the underlying reader, update the hashing function, and
// pass the results through. Returns BadChecksum (instead of EOF) on
// the last read if the checksum doesn't match.
-func (this HashCheckingReader) Read(p []byte) (n int, err error) {
- n, err = this.Reader.Read(p)
+func (hcr HashCheckingReader) Read(p []byte) (n int, err error) {
+ n, err = hcr.Reader.Read(p)
if n > 0 {
- this.Hash.Write(p[:n])
+ hcr.Hash.Write(p[:n])
}
if err == io.EOF {
- sum := this.Hash.Sum(nil)
- if fmt.Sprintf("%x", sum) != this.Check {
+ sum := hcr.Hash.Sum(nil)
+ if fmt.Sprintf("%x", sum) != hcr.Check {
err = BadChecksum
}
}
return n, err
}
-// WriteTo writes the entire contents of this.Reader to dest. Returns
+// WriteTo writes the entire contents of hcr.Reader to dest. Returns
// BadChecksum if writing is successful but the checksum doesn't
// match.
-func (this HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) {
- if writeto, ok := this.Reader.(io.WriterTo); ok {
- written, err = writeto.WriteTo(io.MultiWriter(dest, this.Hash))
+func (hcr HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) {
+ if writeto, ok := hcr.Reader.(io.WriterTo); ok {
+ written, err = writeto.WriteTo(io.MultiWriter(dest, hcr.Hash))
} else {
- written, err = io.Copy(io.MultiWriter(dest, this.Hash), this.Reader)
+ written, err = io.Copy(io.MultiWriter(dest, hcr.Hash), hcr.Reader)
}
if err != nil {
return written, err
}
- sum := this.Hash.Sum(nil)
- if fmt.Sprintf("%x", sum) != this.Check {
+ sum := hcr.Hash.Sum(nil)
+ if fmt.Sprintf("%x", sum) != hcr.Check {
return written, BadChecksum
}
@@ -68,10 +68,10 @@ func (this HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error
// Close reads all remaining data from the underlying Reader and
// returns BadChecksum if the checksum doesn't match. It also closes
// the underlying Reader if it implements io.ReadCloser.
-func (this HashCheckingReader) Close() (err error) {
- _, err = io.Copy(this.Hash, this.Reader)
+func (hcr HashCheckingReader) Close() (err error) {
+ _, err = io.Copy(hcr.Hash, hcr.Reader)
- if closer, ok := this.Reader.(io.Closer); ok {
+ if closer, ok := hcr.Reader.(io.Closer); ok {
closeErr := closer.Close()
if err == nil {
err = closeErr
@@ -80,7 +80,7 @@ func (this HashCheckingReader) Close() (err error) {
if err != nil {
return err
}
- if fmt.Sprintf("%x", this.Hash.Sum(nil)) != this.Check {
+ if fmt.Sprintf("%x", hcr.Hash.Sum(nil)) != hcr.Check {
return BadChecksum
}
return nil
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 59c412724..57a89b50a 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -97,7 +97,7 @@ func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
type StubPutHandler struct {
c *C
expectPath string
- expectApiToken string
+ expectAPIToken string
expectBody string
expectStorageClass string
handled chan string
@@ -105,7 +105,7 @@ type StubPutHandler struct {
func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
- sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
+ sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectAPIToken))
sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
body, err := ioutil.ReadAll(req.Body)
sph.c.Check(err, Equals, nil)
@@ -256,7 +256,7 @@ type KeepServer struct {
func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
ks = make([]KeepServer, n)
- for i := 0; i < n; i += 1 {
+ for i := 0; i < n; i++ {
ks[i] = RunFakeKeepServer(st)
}
@@ -464,14 +464,14 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
type StubGetHandler struct {
c *C
expectPath string
- expectApiToken string
+ expectAPIToken string
httpStatus int
body []byte
}
func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
- sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
+ sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectAPIToken))
resp.WriteHeader(sgh.httpStatus)
resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
resp.Write(sgh.body)
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 91117f2d3..3b1afe1e2 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -55,7 +55,7 @@ type uploadStatus struct {
response string
}
-func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader,
+func (kc *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader,
uploadStatusChan chan<- uploadStatus, expectedLength int64, reqid string) {
var req *http.Request
@@ -77,15 +77,15 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
}
req.Header.Add("X-Request-Id", reqid)
- req.Header.Add("Authorization", "OAuth2 "+this.Arvados.ApiToken)
+ req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
req.Header.Add("Content-Type", "application/octet-stream")
- req.Header.Add(XKeepDesiredReplicas, fmt.Sprint(this.Want_replicas))
- if len(this.StorageClasses) > 0 {
- req.Header.Add("X-Keep-Storage-Classes", strings.Join(this.StorageClasses, ", "))
+ req.Header.Add(XKeepDesiredReplicas, fmt.Sprint(kc.Want_replicas))
+ if len(kc.StorageClasses) > 0 {
+ req.Header.Add("X-Keep-Storage-Classes", strings.Join(kc.StorageClasses, ", "))
}
var resp *http.Response
- if resp, err = this.httpClient().Do(req); err != nil {
+ if resp, err = kc.httpClient().Do(req); err != nil {
DebugPrintf("DEBUG: [%s] Upload failed %v error: %v", reqid, url, err.Error())
uploadStatusChan <- uploadStatus{err, url, 0, 0, err.Error()}
return
@@ -116,15 +116,15 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
}
}
-func (this *KeepClient) putReplicas(
+func (kc *KeepClient) putReplicas(
hash string,
getReader func() io.Reader,
expectedLength int64) (locator string, replicas int, err error) {
- reqid := this.getRequestID()
+ reqid := kc.getRequestID()
// Calculate the ordering for uploading to servers
- sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
+ sv := NewRootSorter(kc.WritableLocalRoots(), hash).GetSortedRoots()
// The next server to try contacting
nextServer := 0
@@ -147,15 +147,15 @@ func (this *KeepClient) putReplicas(
}()
replicasDone := 0
- replicasTodo := this.Want_replicas
+ replicasTodo := kc.Want_replicas
- replicasPerThread := this.replicasPerService
+ replicasPerThread := kc.replicasPerService
if replicasPerThread < 1 {
// unlimited or unknown
replicasPerThread = replicasTodo
}
- retriesRemaining := 1 + this.Retries
+ retriesRemaining := 1 + kc.Retries
var retryServers []string
lastError := make(map[string]string)
@@ -169,7 +169,7 @@ func (this *KeepClient) putReplicas(
// Start some upload requests
if nextServer < len(sv) {
DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[nextServer])
- go this.uploadToKeepServer(sv[nextServer], hash, getReader(), uploadStatusChan, expectedLength, reqid)
+ go kc.uploadToKeepServer(sv[nextServer], hash, getReader(), uploadStatusChan, expectedLength, reqid)
nextServer++
active++
} else {
commit ffc10a3d7cede7651eb1b78353be907cff8758b4
Author: Peter van Heusden <pvh at sanbi.ac.za>
Date: Thu Nov 26 20:35:16 2020 +0200
Fix hardcoded initial user name and email
Arvados-DCO-1.1-Signed-off-by: Peter Van Heusden <vh at sanbi.ac.za>
diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
index 7726d86ec..cf61d92b5 100755
--- a/tools/salt-install/tests/run-test.sh
+++ b/tools/salt-install/tests/run-test.sh
@@ -34,7 +34,7 @@ arv-keepdocker --pull arvados/jobs "${VERSION}" --project-uuid "${project_uuid}"
# Create the initial user
echo "Creating initial user '__INITIAL_USER__'"
-user_uuid=$(arv --format=uuid user list --filters '[["email", "=", "admin at arva2.arv.local"], ["username", "=", "admin"]]')
+user_uuid=$(arv --format=uuid user list --filters '[["email", "=", "__INITIAL_USER_EMAIL__"], ["username", "=", "__INITIAL_USER__"]]')
if [ "x${user_uuid}" = "x" ]; then
user_uuid=$(arv --format=uuid user create --user '{"email": "__INITIAL_USER_EMAIL__", "username": "__INITIAL_USER__"}')
commit cf2d4da8f988451d7a3f1d37f5a12e4fcfc2f0c4
Author: Ward Vandewege <ward at curii.com>
Date: Wed Nov 25 17:24:35 2020 -0500
Fix more golint warnings.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 15ff0607a..342ef03a7 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -293,10 +293,10 @@ rm ${zip}
DataDirectory string
LogFile string
}
- if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
+ if pgLsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
err = fmt.Errorf("pg_lsclusters: %s", err2)
return 1
- } else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 {
+ } else if pgclusters := strings.Split(strings.TrimSpace(string(pgLsclusters)), "\n"); len(pgclusters) != 1 {
logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters))
} else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil {
err = fmt.Errorf("error parsing pg_lsclusters output: %s", err)
diff --git a/sdk/go/blockdigest/blockdigest_test.go b/sdk/go/blockdigest/blockdigest_test.go
index a9994f704..9e8f9a4a0 100644
--- a/sdk/go/blockdigest/blockdigest_test.go
+++ b/sdk/go/blockdigest/blockdigest_test.go
@@ -13,8 +13,8 @@ import (
func getStackTrace() string {
buf := make([]byte, 1000)
- bytes_written := runtime.Stack(buf, false)
- return "Stack Trace:\n" + string(buf[:bytes_written])
+ bytesWritten := runtime.Stack(buf, false)
+ return "Stack Trace:\n" + string(buf[:bytesWritten])
}
func expectEqual(t *testing.T, actual interface{}, expected interface{}) {
commit 3b6a64460d926e198a362293c6dc2b349b142c2d
Author: Javier Bértoli <jbertoli at curii.com>
Date: Tue Nov 24 23:03:37 2020 -0300
fix(test): wrong variable comparison
refs #17146, #17147, #17150
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index a448824a9..ed3466dde 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -33,7 +33,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
arv.vm.provision "shell",
path: "provision.sh",
args: [
- # "--test",
+ "--test",
"--vagrant",
"--ssl-port=8443"
].join(" ")
diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
index a4614aa8b..7726d86ec 100755
--- a/tools/salt-install/tests/run-test.sh
+++ b/tools/salt-install/tests/run-test.sh
@@ -48,7 +48,7 @@ arv user update --uuid "${user_uuid}" --user '{"is_active": true}'
echo "Getting the user API TOKEN"
user_api_token=$(arv api_client_authorization list --filters "[[\"owner_uuid\", \"=\", \"${user_uuid}\"],[\"kind\", \"==\", \"arvados#apiClientAuthorization\"]]" --limit=1 |jq -r .items[].api_token)
-if [ "x${user_uuid}" = "x" ]; then
+if [ "x${user_api_token}" = "x" ]; then
user_api_token=$(arv api_client_authorization create --api-client-authorization "{\"owner_uuid\": \"${user_uuid}\"}" | jq -r .api_token)
fi
commit e40626646448198d33ae7f8f1024fc91c6d1649a
Author: Javier Bértoli <jbertoli at curii.com>
Date: Tue Nov 24 23:02:31 2020 -0300
fix(docker): formula upgraded
refs #17146, #17147, #17150
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 91cb39894..a207d0198 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -168,6 +168,7 @@ cat > ${P_DIR}/top.sls << EOFPSLS
base:
'*':
- arvados
+ - docker
- locale
- nginx_api_configuration
- nginx_controller_configuration
diff --git a/tools/salt-install/single_host/docker.sls b/tools/salt-install/single_host/docker.sls
new file mode 100644
index 000000000..54d225615
--- /dev/null
+++ b/tools/salt-install/single_host/docker.sls
@@ -0,0 +1,9 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+docker:
+ pkg:
+ docker:
+ use_upstream: package
commit 33ef9263f9e574768d95fbc65ac302093a7158ad
Author: Javier Bértoli <jbertoli at curii.com>
Date: Tue Nov 24 17:17:25 2020 -0300
Fix salt-install and test scripts
* add checks to find existing resources
* run some shellcheck
refs #17146, #17147, #17150
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 57a26308e..91cb39894 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash
# Copyright (C) The Arvados Authors. All rights reserved.
#
@@ -55,9 +55,9 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
usage() {
echo >&2
- echo >&2 "Usage: $0 [-h] [-h]"
+ echo >&2 "Usage: ${0} [-h] [-h]"
echo >&2
- echo >&2 "$0 options:"
+ echo >&2 "${0} options:"
echo >&2 " -d, --debug Run salt installation in debug mode"
echo >&2 " -p <N>, --ssl-port <N> SSL port to use for the web applications"
echo >&2 " -t, --test Test installation running a CWL workflow"
@@ -68,16 +68,16 @@ usage() {
arguments() {
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
- TEMP=`getopt -o dhp:tv \
+ TEMP=$(getopt -o dhp:tv \
--long debug,help,ssl-port:,test,vagrant \
- -n "$0" -- "$@"`
+ -n "${0}" -- "${@}")
- if [ $? != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
+ if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"
- while [ $# -ge 1 ]; do
- case $1 in
+ while [ ${#} -ge 1 ]; do
+ case ${1} in
-d | --debug)
LOG_LEVEL="debug"
shift
@@ -110,7 +110,7 @@ LOG_LEVEL="info"
HOST_SSL_PORT=443
TESTS_DIR="tests"
-arguments $@
+arguments ${@}
# Salt's dir
## states
@@ -185,18 +185,15 @@ EOFPSLS
# Get the formula and dependencies
cd ${F_DIR} || exit 1
for f in postgres arvados nginx docker locale; do
- git clone https://github.com/netmanagers/${f}-formula.git
+ git clone https://github.com/saltstack-formulas/${f}-formula.git
done
if [ "x${BRANCH}" != "x" ]; then
- cd ${F_DIR}/arvados-formula
- git checkout -t origin/${BRANCH}
+ cd ${F_DIR}/arvados-formula || exit 1
+ git checkout -t origin/"${BRANCH}"
cd -
fi
-# sed "s/__DOMAIN__/${DOMAIN}/g; s/__CLUSTER__/${CLUSTER}/g; s/__RELEASE__/${RELEASE}/g; s/__VERSION__/${VERSION}/g" \
-# ${CONFIG_DIR}/arvados_dev.sls > ${P_DIR}/arvados.sls
-
if [ "x${VAGRANT}" = "xyes" ]; then
SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}"
TESTS_DIR="/vagrant/${TESTS_DIR}"
@@ -206,7 +203,7 @@ else
fi
# Replace cluster and domain name in the example pillars and test files
-for f in ${SOURCE_PILLARS_DIR}/*; do
+for f in "${SOURCE_PILLARS_DIR}"/*; do
sed "s/__CLUSTER__/${CLUSTER}/g;
s/__DOMAIN__/${DOMAIN}/g;
s/__RELEASE__/${RELEASE}/g;
@@ -216,12 +213,12 @@ for f in ${SOURCE_PILLARS_DIR}/*; do
s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
s/__VERSION__/${VERSION}/g" \
- ${f} > ${P_DIR}/$(basename ${f})
+ "${f}" > "${P_DIR}"/$(basename "${f}")
done
mkdir -p /tmp/cluster_tests
# Replace cluster and domain name in the example pillars and test files
-for f in ${TESTS_DIR}/*; do
+for f in "${TESTS_DIR}"/*; do
sed "s/__CLUSTER__/${CLUSTER}/g;
s/__DOMAIN__/${DOMAIN}/g;
s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
index b91101ee1..a4614aa8b 100755
--- a/tools/salt-install/tests/run-test.sh
+++ b/tools/salt-install/tests/run-test.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env /bin/bash
+#!/bin/bash
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -11,32 +11,49 @@ export ARVADOS_API_HOST_INSECURE=true
# https://doc.arvados.org/v2.0/install/install-jobs-image.html
echo "Creating Arvados Standard Docker Images project"
uuid_prefix=$(arv --format=uuid user current | cut -d- -f1)
-project_uuid=$(arv --format=uuid group create --group "{\"owner_uuid\": \"${uuid_prefix}-tpzed-000000000000000\", \"group_class\":\"project\", \"name\":\"Arvados Standard Docker Images\"}")
-echo "Arvados project uuid is '${project_uuid}'"
-read -rd $'\000' newlink <<EOF; arv link create --link "${newlink}"
+project_uuid=$(arv --format=uuid group list --filters '[["name", "=", "Arvados Standard Docker Images"]]')
+
+if [ "x${project_uuid}" = "x" ]; then
+ project_uuid=$(arv --format=uuid group create --group "{\"owner_uuid\": \"${uuid_prefix}-tpzed-000000000000000\", \"group_class\":\"project\", \"name\":\"Arvados Standard Docker Images\"}")
+
+ read -rd $'\000' newlink <<EOF; arv link create --link "${newlink}"
{
-"tail_uuid":"${uuid_prefix}-j7d0g-fffffffffffffff",
-"head_uuid":"${project_uuid}",
-"link_class":"permission",
-"name":"can_read"
+ "tail_uuid":"${uuid_prefix}-j7d0g-fffffffffffffff",
+ "head_uuid":"${project_uuid}",
+ "link_class":"permission",
+ "name":"can_read"
}
EOF
+fi
+
+echo "Arvados project uuid is '${project_uuid}'"
echo "Uploading arvados/jobs' docker image to the project"
VERSION="2.1.1"
-arv-keepdocker --pull arvados/jobs ${VERSION} --project-uuid ${project_uuid}
+arv-keepdocker --pull arvados/jobs "${VERSION}" --project-uuid "${project_uuid}"
# Create the initial user
-echo "Creating initial user ('__INITIAL_USER__')"
-user=$(arv --format=uuid user create --user '{"email": "__INITIAL_USER_EMAIL__", "username": "__INITIAL_USER__"}')
-echo "Setting up user ('__INITIAL_USER__')"
-arv user setup --uuid ${user}
+echo "Creating initial user '__INITIAL_USER__'"
+user_uuid=$(arv --format=uuid user list --filters '[["email", "=", "admin at arva2.arv.local"], ["username", "=", "admin"]]')
+
+if [ "x${user_uuid}" = "x" ]; then
+ user_uuid=$(arv --format=uuid user create --user '{"email": "__INITIAL_USER_EMAIL__", "username": "__INITIAL_USER__"}')
+ echo "Setting up user '__INITIAL_USER__'"
+ arv user setup --uuid "${user_uuid}"
+fi
+
echo "Activating user '__INITIAL_USER__'"
-arv user update --uuid ${user} --user '{"is_active": true}'
+arv user update --uuid "${user_uuid}" --user '{"is_active": true}'
-user_api_token=$(arv api_client_authorization create --api-client-authorization "{\"owner_uuid\": \"${user}\"}" | jq -r .api_token)
+echo "Getting the user API TOKEN"
+user_api_token=$(arv api_client_authorization list --filters "[[\"owner_uuid\", \"=\", \"${user_uuid}\"],[\"kind\", \"==\", \"arvados#apiClientAuthorization\"]]" --limit=1 |jq -r .items[].api_token)
+
+if [ "x${user_uuid}" = "x" ]; then
+ user_api_token=$(arv api_client_authorization create --api-client-authorization "{\"owner_uuid\": \"${user_uuid}\"}" | jq -r .api_token)
+fi
-echo "Running test CWL workflow"
# Change to the user's token and run the workflow
-export ARVADOS_API_TOKEN=${user_api_token}
+export ARVADOS_API_TOKEN="${user_api_token}"
+
+echo "Running test CWL workflow"
cwl-runner hasher-workflow.cwl hasher-workflow-job.yml
commit e916ddcdf18b8c9aad35c2a810c4636ad805e7bf
Author: Peter van Heusden <pvh at sanbi.ac.za>
Date: Wed Nov 18 16:01:32 2020 +0200
Fix salt-install's crunch-dispatch-local config and tests
* added [@pvanheus fix]:https://github.com/arvados/arvados/pull/140
* added [a debug parameter to the provision script](https://gitter.im/arvados/community?at=5fb54b4bd37a1a13d6b46a05)
* added a test script to verify the cluster is able to run a CWL workflow
* document salt-install's workflow test
refs #17146, #17147, #17150
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/doc/install/salt-single-host.html.textile.liquid b/doc/install/salt-single-host.html.textile.liquid
index 139366179..fb41d59ee 100644
--- a/doc/install/salt-single-host.html.textile.liquid
+++ b/doc/install/salt-single-host.html.textile.liquid
@@ -11,9 +11,9 @@ SPDX-License-Identifier: CC-BY-SA-3.0
# "Install Saltstack":#saltstack
# "Single host install using the provision.sh script":#single_host
-# "Local testing Arvados in a Vagrant box":#vagrant
# "DNS configuration":#final_steps
# "Initial user and login":#initial_user
+# "Test the installed cluster running a simple workflow":#test_install
h2(#saltstack). Install Saltstack
@@ -84,3 +84,95 @@ Assuming you didn't change these values in the @provision.sh@ script, the initia
* User: 'admin'
* Password: 'password'
* Email: 'admin at arva2.arv.local'
+
+h2(#test_install). Test the installed cluster running a simple workflow
+
+The @provision.sh@ script saves a simple example test workflow in the @/tmp/cluster_tests at . If you want to run it, just change to that directory and run:
+
+<notextile>
+<pre><code>cd /tmp/cluster_tests
+./run-test.sh
+</code></pre>
+</notextile>
+
+It will create a test user, upload a small workflow and run it. If everything goes OK, the output should similar to this (some output was shortened for clarity):
+
+<notextile>
+<pre><code>Creating Arvados Standard Docker Images project
+Arvados project uuid is 'arva2-j7d0g-0prd8cjlk6kfl7y'
+{
+ ...
+ "uuid":"arva2-o0j2j-n4zu4cak5iifq2a",
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+}
+Uploading arvados/jobs' docker image to the project
+2.1.1: Pulling from arvados/jobs
+8559a31e96f4: Pulling fs layer
+...
+Status: Downloaded newer image for arvados/jobs:2.1.1
+docker.io/arvados/jobs:2.1.1
+2020-11-23 21:43:39 arvados.arv_put[32678] INFO: Creating new cache file at /home/vagrant/.cache/arvados/arv-put/c59256eda1829281424c80f588c7cc4d
+2020-11-23 21:43:46 arvados.arv_put[32678] INFO: Collection saved as 'Docker image arvados jobs:2.1.1 sha256:0dd50'
+arva2-4zz18-1u5pvbld7cvxuy2
+Creating initial user ('admin')
+Setting up user ('admin')
+{
+ "items":[
+ {
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "uuid":"arva2-o0j2j-1ownrdne0ok9iox"
+ },
+ {
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "uuid":"arva2-o0j2j-1zbeyhcwxc1tvb7"
+ },
+ {
+ ...
+ "email":"admin at arva2.arv.local",
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "username":"admin",
+ "uuid":"arva2-tpzed-3wrm93zmzpshrq2",
+ ...
+ }
+ ],
+ "kind":"arvados#HashList"
+}
+Activating user 'admin'
+{
+ ...
+ "email":"admin at arva2.arv.local",
+ ...
+ "username":"admin",
+ "uuid":"arva2-tpzed-3wrm93zmzpshrq2",
+ ...
+}
+Running test CWL workflow
+INFO /usr/bin/cwl-runner 2.1.1, arvados-python-client 2.1.1, cwltool 3.0.20200807132242
+INFO Resolved 'hasher-workflow.cwl' to 'file:///tmp/cluster_tests/hasher-workflow.cwl'
+...
+INFO Using cluster arva2 (https://arva2.arv.local:8443/)
+INFO Upload local files: "test.txt"
+INFO Uploaded to ea34d971b71d5536b4f6b7d6c69dc7f6+50 (arva2-4zz18-c8uvwqdry4r8jao)
+INFO Using collection cache size 256 MiB
+INFO [container hasher-workflow.cwl] submitted container_request arva2-xvhdp-v1bkywd58gyocwm
+INFO [container hasher-workflow.cwl] arva2-xvhdp-v1bkywd58gyocwm is Final
+INFO Overall process status is success
+INFO Final output collection d6c69a88147dde9d52a418d50ef788df+123
+{
+ "hasher_out": {
+ "basename": "hasher3.md5sum.txt",
+ "class": "File",
+ "location": "keep:d6c69a88147dde9d52a418d50ef788df+123/hasher3.md5sum.txt",
+ "size": 95
+ }
+}
+INFO Final process status is success
+</code></pre>
+</notextile>
diff --git a/doc/install/salt-vagrant.html.textile.liquid b/doc/install/salt-vagrant.html.textile.liquid
index 41f32e51c..d9aa791f0 100644
--- a/doc/install/salt-vagrant.html.textile.liquid
+++ b/doc/install/salt-vagrant.html.textile.liquid
@@ -12,6 +12,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
# "Vagrant":#vagrant
# "DNS configuration":#final_steps
# "Initial user and login":#initial_user
+# "Test the installed cluster running a simple workflow":#test_install
h2(#vagrant). Vagrant
@@ -71,3 +72,19 @@ Assuming you didn't change the defaults, the initial credentials are:
* User: 'admin'
* Password: 'password'
* Email: 'admin at arva2.arv.local'
+
+h2(#test_install). Test the installed cluster running a simple workflow
+
+As documented in the <a href="{{ site.baseurl }}/install/salt-single-host.html">Single Host installation</a> page, You can run a test workflow to verify the installation finished correctly. To do so, you can follow these steps:
+
+<notextile>
+<pre><code>vagrant ssh</code></pre>
+</notextile>
+
+and once in the instance:
+
+<notextile>
+<pre><code>cd /tmp/cluster_tests
+./run-test.sh
+</code></pre>
+</notextile>
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index 93bb77d4f..a448824a9 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -13,7 +13,13 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.define "arvados" do |arv|
arv.vm.box = "bento/debian-10"
- arv.vm.hostname = "arva2.arv.local"
+ arv.vm.hostname = "vagrant.local"
+ # CPU/RAM
+ config.vm.provider :virtualbox do |v|
+ v.memory = 2048
+ v.cpus = 2
+ end
+
# Networking
arv.vm.network "forwarded_port", guest: 8443, host: 8443
arv.vm.network "forwarded_port", guest: 25100, host: 25100
@@ -24,12 +30,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
arv.vm.network "forwarded_port", guest: 8001, host: 8001
arv.vm.network "forwarded_port", guest: 8000, host: 8000
arv.vm.network "forwarded_port", guest: 3001, host: 3001
- # config.vm.network "private_network", ip: "192.168.33.10"
- # arv.vm.synced_folder "salt_pillars", "/srv/pillars",
- # create: true
arv.vm.provision "shell",
path: "provision.sh",
args: [
+ # "--test",
"--vagrant",
"--ssl-port=8443"
].join(" ")
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 7e88d7662..57a26308e 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -50,21 +50,26 @@ VERSION="latest"
set -o pipefail
+# capture the directory that the script is running from
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
usage() {
echo >&2
echo >&2 "Usage: $0 [-h] [-h]"
echo >&2
echo >&2 "$0 options:"
- echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
+ echo >&2 " -d, --debug Run salt installation in debug mode"
echo >&2 " -p <N>, --ssl-port <N> SSL port to use for the web applications"
+ echo >&2 " -t, --test Test installation running a CWL workflow"
echo >&2 " -h, --help Display this help and exit"
+ echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
echo >&2
}
arguments() {
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
- TEMP=`getopt -o hvp: \
- --long help,vagrant,ssl-port: \
+ TEMP=`getopt -o dhp:tv \
+ --long debug,help,ssl-port:,test,vagrant \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
@@ -73,6 +78,14 @@ arguments() {
while [ $# -ge 1 ]; do
case $1 in
+ -d | --debug)
+ LOG_LEVEL="debug"
+ shift
+ ;;
+ -t | --test)
+ TEST="yes"
+ shift
+ ;;
-v | --vagrant)
VAGRANT="yes"
shift
@@ -93,7 +106,9 @@ arguments() {
done
}
+LOG_LEVEL="info"
HOST_SSL_PORT=443
+TESTS_DIR="tests"
arguments $@
@@ -106,7 +121,7 @@ F_DIR="/srv/formulas"
P_DIR="/srv/pillars"
apt-get update
-apt-get install -y curl git
+apt-get install -y curl git jq
dpkg -l |grep salt-minion
if [ ${?} -eq 0 ]; then
@@ -139,6 +154,7 @@ mkdir -p ${P_DIR}
cat > ${S_DIR}/top.sls << EOFTSLS
base:
'*':
+ - example_single_host_host_entries
- example_add_snakeoil_certs
- locale
- nginx.passenger
@@ -169,7 +185,7 @@ EOFPSLS
# Get the formula and dependencies
cd ${F_DIR} || exit 1
for f in postgres arvados nginx docker locale; do
- git clone https://github.com/saltstack-formulas/${f}-formula.git
+ git clone https://github.com/netmanagers/${f}-formula.git
done
if [ "x${BRANCH}" != "x" ]; then
@@ -183,15 +199,16 @@ fi
if [ "x${VAGRANT}" = "xyes" ]; then
SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}"
+ TESTS_DIR="/vagrant/${TESTS_DIR}"
else
- SOURCE_PILLARS_DIR="./${CONFIG_DIR}"
+ SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}"
+ TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
fi
-# Replace cluster and domain name in the example pillars
+# Replace cluster and domain name in the example pillars and test files
for f in ${SOURCE_PILLARS_DIR}/*; do
- # sed "s/example.net/${DOMAIN}/g; s/fixme/${CLUSTER}/g" \
- sed "s/__DOMAIN__/${DOMAIN}/g;
- s/__CLUSTER__/${CLUSTER}/g;
+ sed "s/__CLUSTER__/${CLUSTER}/g;
+ s/__DOMAIN__/${DOMAIN}/g;
s/__RELEASE__/${RELEASE}/g;
s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
s/__GUEST_SSL_PORT__/${GUEST_SSL_PORT}/g;
@@ -202,9 +219,18 @@ for f in ${SOURCE_PILLARS_DIR}/*; do
${f} > ${P_DIR}/$(basename ${f})
done
-# Let's write an /etc/hosts file that points all the hosts to localhost
-
-echo "127.0.0.2 api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+mkdir -p /tmp/cluster_tests
+# Replace cluster and domain name in the example pillars and test files
+for f in ${TESTS_DIR}/*; do
+ sed "s/__CLUSTER__/${CLUSTER}/g;
+ s/__DOMAIN__/${DOMAIN}/g;
+ s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
+ s/__INITIAL_USER__/${INITIAL_USER}/g;
+ s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
+ s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g" \
+ ${f} > /tmp/cluster_tests/$(basename ${f})
+done
+chmod 755 /tmp/cluster_tests/run-test.sh
# FIXME! #16992 Temporary fix for psql call in arvados-api-server
if [ -e /root/.psqlrc ]; then
@@ -220,7 +246,7 @@ echo '\pset pager off' >> /root/.psqlrc
# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
# Now run the install
-salt-call --local state.apply -l debug
+salt-call --local state.apply -l ${LOG_LEVEL}
# FIXME! #16992 Temporary fix for psql call in arvados-api-server
if [ "x${DELETE_PSQL}" = "xyes" ]; then
@@ -229,7 +255,18 @@ if [ "x${DELETE_PSQL}" = "xyes" ]; then
fi
if [ "x${RESTORE_PSQL}" = "xyes" ]; then
- echo "Restroting .psql file"
+ echo "Restoring .psql file"
mv -v /root/.psqlrc.provision.backup /root/.psqlrc
fi
# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
+
+# If running in a vagrant VM, add default user to docker group
+if [ "x${VAGRANT}" = "xyes" ]; then
+ usermod -a -G docker vagrant
+fi
+
+# Test that the installation finished correctly
+if [ "x${TEST}" = "xyes" ]; then
+ cd /tmp/cluster_tests
+ ./run-test.sh
+fi
diff --git a/tools/salt-install/single_host/arvados.sls b/tools/salt-install/single_host/arvados.sls
index ad0cbab70..dffd6575e 100644
--- a/tools/salt-install/single_host/arvados.sls
+++ b/tools/salt-install/single_host/arvados.sls
@@ -78,19 +78,19 @@ arvados:
### TOKENS
tokens:
- system_root: changeme_system_root_token
- management: changeme_management_token
- rails_secret: changeme_rails_secret_token
- anonymous_user: changeme_anonymous_user_token
+ system_root: changemesystemroottoken
+ management: changememanagementtoken
+ rails_secret: changemerailssecrettoken
+ anonymous_user: changemeanonymoususertoken
### KEYS
secrets:
- blob_signing_key: changeme_blob_signing_key
- workbench_secret_key: changeme_workbench_secret_key
- dispatcher_access_key: changeme_dispatcher_access_key
- dispatcher_secret_key: changeme_dispatcher_secret_key
- keep_access_key: changeme_keep_access_key
- keep_secret_key: changeme_keep_secret_key
+ blob_signing_key: changemeblobsigningkey
+ workbench_secret_key: changemeworkbenchsecretkey
+ dispatcher_access_key: changemedispatcheraccesskey
+ dispatcher_secret_key: changeme_dispatchersecretkey
+ keep_access_key: changemekeepaccesskey
+ keep_secret_key: changemekeepsecretkey
Login:
Test:
@@ -124,7 +124,7 @@ arvados:
Controller:
ExternalURL: https://__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
InternalURLs:
- http://127.0.0.2:8003: {}
+ http://controller.internal:8003: {}
DispatchCloud:
InternalURLs:
http://__CLUSTER__.__DOMAIN__:9006: {}
@@ -134,17 +134,17 @@ arvados:
Keepproxy:
ExternalURL: https://keep.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
InternalURLs:
- http://127.0.0.2:25100: {}
+ http://keep.internal:25100: {}
Keepstore:
InternalURLs:
http://keep0.__CLUSTER__.__DOMAIN__:25107: {}
RailsAPI:
InternalURLs:
- http://127.0.0.2:8004: {}
+ http://api.internal:8004: {}
WebDAV:
ExternalURL: https://collections.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
InternalURLs:
- http://127.0.0.2:9002: {}
+ http://collections.internal:9002: {}
WebDAVDownload:
ExternalURL: https://download.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
WebShell:
@@ -152,7 +152,7 @@ arvados:
Websocket:
ExternalURL: wss://ws.__CLUSTER__.__DOMAIN__/websocket
InternalURLs:
- http://127.0.0.2:8005: {}
+ http://ws.internal:8005: {}
Workbench1:
ExternalURL: https://workbench.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
Workbench2:
diff --git a/tools/salt-install/single_host/nginx_api_configuration.sls b/tools/salt-install/single_host/nginx_api_configuration.sls
index db0bea126..b2f12c773 100644
--- a/tools/salt-install/single_host/nginx_api_configuration.sls
+++ b/tools/salt-install/single_host/nginx_api_configuration.sls
@@ -18,7 +18,7 @@ nginx:
overwrite: true
config:
- server:
- - listen: '127.0.0.2:8004'
+ - listen: 'api.internal:8004'
- server_name: api
- root: /var/www/arvados-api/current/public
- index: index.html index.htm
diff --git a/tools/salt-install/single_host/nginx_controller_configuration.sls b/tools/salt-install/single_host/nginx_controller_configuration.sls
index 2b2e7d591..7c99d2dea 100644
--- a/tools/salt-install/single_host/nginx_controller_configuration.sls
+++ b/tools/salt-install/single_host/nginx_controller_configuration.sls
@@ -14,7 +14,7 @@ nginx:
default: 1
'127.0.0.0/8': 0
upstream controller_upstream:
- - server: '127.0.0.2:8003 fail_timeout=10s'
+ - server: 'controller.internal:8003 fail_timeout=10s'
### SITES
servers:
diff --git a/tools/salt-install/single_host/nginx_keepproxy_configuration.sls b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
index 29cd0cb44..fc4854e5a 100644
--- a/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
+++ b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
@@ -11,7 +11,7 @@ nginx:
### STREAMS
http:
upstream keepproxy_upstream:
- - server: '127.0.0.2:25100 fail_timeout=10s'
+ - server: 'keep.internal:25100 fail_timeout=10s'
servers:
managed:
diff --git a/tools/salt-install/single_host/nginx_keepweb_configuration.sls b/tools/salt-install/single_host/nginx_keepweb_configuration.sls
index bd0a636b0..513c0393e 100644
--- a/tools/salt-install/single_host/nginx_keepweb_configuration.sls
+++ b/tools/salt-install/single_host/nginx_keepweb_configuration.sls
@@ -11,7 +11,7 @@ nginx:
### STREAMS
http:
upstream collections_downloads_upstream:
- - server: '127.0.0.2:9002 fail_timeout=10s'
+ - server: 'collections.internal:9002 fail_timeout=10s'
servers:
managed:
diff --git a/tools/salt-install/single_host/nginx_webshell_configuration.sls b/tools/salt-install/single_host/nginx_webshell_configuration.sls
index e33ddcea7..495de82d2 100644
--- a/tools/salt-install/single_host/nginx_webshell_configuration.sls
+++ b/tools/salt-install/single_host/nginx_webshell_configuration.sls
@@ -12,7 +12,7 @@ nginx:
### STREAMS
http:
upstream webshell_upstream:
- - server: '127.0.0.2:4200 fail_timeout=10s'
+ - server: 'shell.internal:4200 fail_timeout=10s'
### SITES
servers:
diff --git a/tools/salt-install/single_host/nginx_websocket_configuration.sls b/tools/salt-install/single_host/nginx_websocket_configuration.sls
index 2241d3b8e..1848a8737 100644
--- a/tools/salt-install/single_host/nginx_websocket_configuration.sls
+++ b/tools/salt-install/single_host/nginx_websocket_configuration.sls
@@ -11,7 +11,7 @@ nginx:
### STREAMS
http:
upstream websocket_upstream:
- - server: '127.0.0.2:8005 fail_timeout=10s'
+ - server: 'ws.internal:8005 fail_timeout=10s'
servers:
managed:
diff --git a/tools/salt-install/single_host/nginx_workbench_configuration.sls b/tools/salt-install/single_host/nginx_workbench_configuration.sls
index 76fb13438..9a382e777 100644
--- a/tools/salt-install/single_host/nginx_workbench_configuration.sls
+++ b/tools/salt-install/single_host/nginx_workbench_configuration.sls
@@ -17,7 +17,7 @@ nginx:
### STREAMS
http:
upstream workbench_upstream:
- - server: '127.0.0.2:9000 fail_timeout=10s'
+ - server: 'workbench.internal:9000 fail_timeout=10s'
### SITES
servers:
@@ -64,7 +64,7 @@ nginx:
overwrite: true
config:
- server:
- - listen: '127.0.0.2:9000'
+ - listen: 'workbench.internal:9000'
- server_name: workbench
- root: /var/www/arvados-workbench/current/public
- index: index.html index.htm
diff --git a/tools/salt-install/tests/hasher-workflow-job.yml b/tools/salt-install/tests/hasher-workflow-job.yml
new file mode 100644
index 000000000..8e5f61167
--- /dev/null
+++ b/tools/salt-install/tests/hasher-workflow-job.yml
@@ -0,0 +1,10 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+inputfile:
+ class: File
+ path: test.txt
+hasher1_outputname: hasher1.md5sum.txt
+hasher2_outputname: hasher2.md5sum.txt
+hasher3_outputname: hasher3.md5sum.txt
diff --git a/tools/salt-install/tests/hasher-workflow.cwl b/tools/salt-install/tests/hasher-workflow.cwl
new file mode 100644
index 000000000..a23a22f91
--- /dev/null
+++ b/tools/salt-install/tests/hasher-workflow.cwl
@@ -0,0 +1,65 @@
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+
+inputs:
+ inputfile: File
+ hasher1_outputname: string
+ hasher2_outputname: string
+ hasher3_outputname: string
+
+outputs:
+ hasher_out:
+ type: File
+ outputSource: hasher3/hasher_out
+
+steps:
+ hasher1:
+ run: hasher.cwl
+ in:
+ inputfile: inputfile
+ outputname: hasher1_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
+
+ hasher2:
+ run: hasher.cwl
+ in:
+ inputfile: hasher1/hasher_out
+ outputname: hasher2_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
+
+ hasher3:
+ run: hasher.cwl
+ in:
+ inputfile: hasher2/hasher_out
+ outputname: hasher3_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
diff --git a/tools/salt-install/tests/hasher.cwl b/tools/salt-install/tests/hasher.cwl
new file mode 100644
index 000000000..0a0f64f05
--- /dev/null
+++ b/tools/salt-install/tests/hasher.cwl
@@ -0,0 +1,24 @@
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+
+baseCommand: md5sum
+inputs:
+ inputfile:
+ type: File
+ inputBinding:
+ position: 1
+ outputname:
+ type: string
+
+stdout: $(inputs.outputname)
+
+outputs:
+ hasher_out:
+ type: File
+ outputBinding:
+ glob: $(inputs.outputname)
diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
new file mode 100755
index 000000000..b91101ee1
--- /dev/null
+++ b/tools/salt-install/tests/run-test.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env /bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+export ARVADOS_API_TOKEN=changemesystemroottoken
+export ARVADOS_API_HOST=__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+export ARVADOS_API_HOST_INSECURE=true
+
+
+# https://doc.arvados.org/v2.0/install/install-jobs-image.html
+echo "Creating Arvados Standard Docker Images project"
+uuid_prefix=$(arv --format=uuid user current | cut -d- -f1)
+project_uuid=$(arv --format=uuid group create --group "{\"owner_uuid\": \"${uuid_prefix}-tpzed-000000000000000\", \"group_class\":\"project\", \"name\":\"Arvados Standard Docker Images\"}")
+echo "Arvados project uuid is '${project_uuid}'"
+read -rd $'\000' newlink <<EOF; arv link create --link "${newlink}"
+{
+"tail_uuid":"${uuid_prefix}-j7d0g-fffffffffffffff",
+"head_uuid":"${project_uuid}",
+"link_class":"permission",
+"name":"can_read"
+}
+EOF
+
+echo "Uploading arvados/jobs' docker image to the project"
+VERSION="2.1.1"
+arv-keepdocker --pull arvados/jobs ${VERSION} --project-uuid ${project_uuid}
+
+# Create the initial user
+echo "Creating initial user ('__INITIAL_USER__')"
+user=$(arv --format=uuid user create --user '{"email": "__INITIAL_USER_EMAIL__", "username": "__INITIAL_USER__"}')
+echo "Setting up user ('__INITIAL_USER__')"
+arv user setup --uuid ${user}
+echo "Activating user '__INITIAL_USER__'"
+arv user update --uuid ${user} --user '{"is_active": true}'
+
+user_api_token=$(arv api_client_authorization create --api-client-authorization "{\"owner_uuid\": \"${user}\"}" | jq -r .api_token)
+
+echo "Running test CWL workflow"
+# Change to the user's token and run the workflow
+export ARVADOS_API_TOKEN=${user_api_token}
+cwl-runner hasher-workflow.cwl hasher-workflow-job.yml
diff --git a/tools/salt-install/tests/test.txt b/tools/salt-install/tests/test.txt
new file mode 100644
index 000000000..a9c439556
--- /dev/null
+++ b/tools/salt-install/tests/test.txt
@@ -0,0 +1,5 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+test
commit e8ad88b12724d5102b5e0dde85014fd86c6827ef
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Nov 25 12:14:34 2020 -0500
17009: Fix bucket-level ops using virtual host-style requests.
refs #17009
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 4ee69f277..373fd9a25 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -225,7 +225,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
fspath := "/by_id"
if id := parseCollectionIDFromDNSName(r.Host); id != "" {
fspath += "/" + id
- objectNameGiven = true
+ objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
} else {
objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
}
commit 7e7aaec01af5c9b2547478529812a66ea4af6bbf
Author: Ward Vandewege <ward at curii.com>
Date: Tue Nov 24 13:17:44 2020 -0500
Documentation cleanups.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/admin/config.html.textile.liquid b/doc/admin/config.html.textile.liquid
index 316b6f48b..745cd2853 100644
--- a/doc/admin/config.html.textile.liquid
+++ b/doc/admin/config.html.textile.liquid
@@ -10,7 +10,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The master Arvados configuration is stored at @/etc/arvados/config.yml@
+The Arvados configuration is stored at @/etc/arvados/config.yml@
See "Migrating Configuration":config-migration.html for information about migrating from legacy component-specific configuration files.
diff --git a/doc/admin/federation.html.textile.liquid b/doc/admin/federation.html.textile.liquid
index eb4a451a8..d6ffb48f4 100644
--- a/doc/admin/federation.html.textile.liquid
+++ b/doc/admin/federation.html.textile.liquid
@@ -57,9 +57,9 @@ Clusters:
LoginCluster: clsr1
</pre>
-The @LoginCluster@ configuration redirects all user logins to the LoginCluster, and the LoginCluster will issue API tokens which will be accepted by the federation. Users are activated or deactivated across the entire federation based on their status on the master cluster.
+The @LoginCluster@ configuration redirects all user logins to the LoginCluster, and the LoginCluster will issue API tokens which will be accepted by the federation. Users are activated or deactivated across the entire federation based on their status on the login cluster.
-Note: tokens issued by the master cluster need to be periodically re-validated when used on other clusters in the federation. The period between revalidation attempts is configured with @Login.RemoteTokenRefresh at . The default is 5 minutes. A longer period reduces overhead from validating tokens, but means it may take longer for other clusters to notice when a token has been revoked or a user has changed status (being activated/deactivated, admin flag changed).
+Note: tokens issued by the login cluster need to be periodically re-validated when used on other clusters in the federation. The period between revalidation attempts is configured with @Login.RemoteTokenRefresh at . The default is 5 minutes. A longer period reduces overhead from validating tokens, but means it may take longer for other clusters to notice when a token has been revoked or a user has changed status (being activated/deactivated, admin flag changed).
To migrate users of existing clusters with separate user databases to use a single LoginCluster, use "arv-federation-migrate":merge-remote-account.html .
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index e8cde5ace..3f622112e 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -35,7 +35,7 @@ TODO: extract this information based on git commit messages and generate changel
<div class="releasenotes">
</notextile>
-h2(#master). development master (as of 2020-10-28)
+h2(#main). development main (as of 2020-10-28)
"Upgrading from 2.1.0":#v2_1_0
diff --git a/doc/api/methods.html.textile.liquid b/doc/api/methods.html.textile.liquid
index 872a1bca7..ae96d0a3b 100644
--- a/doc/api/methods.html.textile.liquid
+++ b/doc/api/methods.html.textile.liquid
@@ -103,7 +103,7 @@ table(table table-bordered table-condensed).
|@=@, @!=@|string, number, timestamp, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@ @["tail_uuid","!=",null]@|
|@<@, @<=@, @>=@, @>@|string, number, or timestamp|Ordering comparison|@["script_version",">","123"]@|
|@like@, @ilike@|string|SQL pattern match. Single character match is @_@ and wildcard is @%@. The @ilike@ operator is case-insensitive|@["script_version","like","d00220fb%"]@|
-|@in@, @not in@|array of strings|Set membership|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
+|@in@, @not in@|array of strings|Set membership|@["script_version","in",["main","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
diff --git a/doc/api/methods/jobs.html.textile.liquid b/doc/api/methods/jobs.html.textile.liquid
index 13fa83876..aa7a58898 100644
--- a/doc/api/methods/jobs.html.textile.liquid
+++ b/doc/api/methods/jobs.html.textile.liquid
@@ -57,7 +57,7 @@ See "Specifying Git versions":#script_version below for more detail about accept
h3(#script_version). Specifying Git versions
-The script_version attribute and arvados_sdk_version runtime constraint are typically given as a branch, tag, or commit hash, but there are many more ways to specify a Git commit. The "specifying revisions" section of the "gitrevisions manual page":http://git-scm.com/docs/gitrevisions.html has a definitive list. Arvados accepts Git versions in any format listed there that names a single commit (not a tree, a blob, or a range of commits). However, some kinds of names can be expected to resolve differently in Arvados than they do in your local repository. For example, <code>HEAD@{1}</code> refers to the local reflog, and @origin/master@ typically refers to a remote branch: neither is likely to work as desired if given as a Git version.
+The script_version attribute and arvados_sdk_version runtime constraint are typically given as a branch, tag, or commit hash, but there are many more ways to specify a Git commit. The "specifying revisions" section of the "gitrevisions manual page":http://git-scm.com/docs/gitrevisions.html has a definitive list. Arvados accepts Git versions in any format listed there that names a single commit (not a tree, a blob, or a range of commits). However, some kinds of names can be expected to resolve differently in Arvados than they do in your local repository. For example, <code>HEAD@{1}</code> refers to the local reflog, and @origin/main@ typically refers to a remote branch: neither is likely to work as desired if given as a Git version.
h3. Runtime constraints
@@ -138,14 +138,14 @@ notextile. <div class="spaced-out">
h4. Examples
-Run the script "crunch_scripts/hash.py" in the repository "you" using the "master" commit. Arvados should re-use a previous job if the script_version of the previous job is the same as the current "master" commit. This works irrespective of whether the previous job was submitted using the name "master", a different branch name or tag indicating the same commit, a SHA-1 commit hash, etc.
+Run the script "crunch_scripts/hash.py" in the repository "you" using the "main" commit. Arvados should re-use a previous job if the script_version of the previous job is the same as the current "main" commit. This works irrespective of whether the previous job was submitted using the name "main", a different branch name or tag indicating the same commit, a SHA-1 commit hash, etc.
<notextile><pre>
{
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
@@ -170,14 +170,14 @@ Run using exactly the version "d00220fb38d4b85ca8fc28a8151702a2b9d1dec5". Arvado
}
</pre></notextile>
-Arvados should re-use a previous job if the "script_version" of the previous job is between "earlier_version_tag" and the "master" commit (inclusive), but not the commit indicated by "blacklisted_version_tag". If there are no previous jobs matching these criteria, run the job using the "master" commit.
+Arvados should re-use a previous job if the "script_version" of the previous job is between "earlier_version_tag" and the "main" commit (inclusive), but not the commit indicated by "blacklisted_version_tag". If there are no previous jobs matching these criteria, run the job using the "main" commit.
<notextile><pre>
{
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
@@ -195,7 +195,7 @@ The same behavior, using filters:
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
@@ -208,14 +208,14 @@ The same behavior, using filters:
}
</pre></notextile>
-Run the script "crunch_scripts/monte-carlo.py" in the repository "you/you" using the current "master" commit. Because it is marked as "nondeterministic", this job will not be considered as a suitable candidate for future job submissions that use the "find_or_create" feature.
+Run the script "crunch_scripts/monte-carlo.py" in the repository "you/you" using the current "main" commit. Because it is marked as "nondeterministic", this job will not be considered as a suitable candidate for future job submissions that use the "find_or_create" feature.
<notextile><pre>
{
"job": {
"script": "monte-carlo.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"nondeterministic": true,
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
diff --git a/doc/api/methods/pipeline_templates.html.textile.liquid b/doc/api/methods/pipeline_templates.html.textile.liquid
index 40297aa05..141072c51 100644
--- a/doc/api/methods/pipeline_templates.html.textile.liquid
+++ b/doc/api/methods/pipeline_templates.html.textile.liquid
@@ -77,7 +77,7 @@ This is a pipeline named "Filter MD5 hash values" with two components, "do_hash"
"do_hash": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"required": true,
@@ -90,7 +90,7 @@ This is a pipeline named "Filter MD5 hash values" with two components, "do_hash"
"filter": {
"script": "0-filter.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "do_hash"
@@ -110,13 +110,13 @@ This pipeline consists of three components. The components "thing1" and "thing2
"cat_in_the_hat": {
"script": "cat.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"thing1": {
"script": "thing1.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "cat_in_the_hat"
@@ -126,7 +126,7 @@ This pipeline consists of three components. The components "thing1" and "thing2
"thing2": {
"script": "thing2.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "cat_in_the_hat"
@@ -146,19 +146,19 @@ This pipeline consists of three components. The component "cleanup" depends on
"thing1": {
"script": "thing1.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"thing2": {
"script": "thing2.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"cleanup": {
"script": "cleanup.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"mess1": {
"output_of": "thing1"
diff --git a/doc/user/tutorials/wgs-tutorial.html.textile.liquid b/doc/user/tutorials/wgs-tutorial.html.textile.liquid
index cd4d1cc71..a68d7ca21 100644
--- a/doc/user/tutorials/wgs-tutorial.html.textile.liquid
+++ b/doc/user/tutorials/wgs-tutorial.html.textile.liquid
@@ -245,9 +245,9 @@ node.json gives a high level overview about the instance such as name, price, an
** Contains about resource consumption (RAM, cpu, disk, network) on the node while it was running
This is different from the log crunchstat.txt because it includes resource consumption of Arvados components that run on the node outside the container such as crunch-run and other processes related to the Keep file system.
-For the highest level logs, the logs are tracking the container that ran the @arvados-cwl-runner@ process which you can think of as the “mastermind” behind tracking which parts of the CWL workflow need to be run when, which have been run already, what order they need to be run, which can be run simultaneously, and so forth and then sending out the related container requests. Each step then has their own logs related to containers running a CWL step of the workflow including a log of standard error that contains the standard error of the code run in that CWL step. Those logs can be found by expanding the steps and clicking on the link to the log collection.
+For the highest level logs, the logs are tracking the container that ran the @arvados-cwl-runner@ process which you can think of as the “workflow runner”. It tracks which parts of the CWL workflow need to be run when, which have been run already, what order they need to be run, which can be run simultaneously, and so forth and then creates the necessary container requests. Each step has its own logs related to containers running a CWL step of the workflow including a log of standard error that contains the standard error of the code run in that CWL step. Those logs can be found by expanding the steps and clicking on the link to the log collection.
-Let’s take a peek at a few of these logs to get you more familiar with them. First, we can look at the @stderr.txt@ of the highest level process. Again recall this should be of the “mastermind” @arvados-cwl-runner@ process. You can click on the log to download it to your local machine, and when you look at the contents - you should see something like the following...
+Let’s take a peek at a few of these logs to get you more familiar with them. First, we can look at the @stderr.txt@ of the highest level process. Again recall this should be of the “workflow runner” @arvados-cwl-runner@ process. You can click on the log to download it to your local machine, and when you look at the contents - you should see something like the following...
<pre><code>2020-06-22T20:30:04.737703197Z INFO /usr/bin/arvados-cwl-runner 2.0.3, arvados-python-client 2.0.3, cwltool 1.0.20190831161204
2020-06-22T20:30:04.743250012Z INFO Resolved '/var/lib/cwl/workflow.json#main' to 'file:///var/lib/cwl/workflow.json#main'
commit 16f07140d31504341faf1158c4b8eccf1771bd7a
Author: Ward Vandewege <ward at curii.com>
Date: Tue Nov 24 13:17:06 2020 -0500
Make run-build-packages-python-and-ruby.sh say so when it decides to
skip building Python packages.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/run-build-packages-python-and-ruby.sh b/build/run-build-packages-python-and-ruby.sh
index 84da280d7..f25530760 100755
--- a/build/run-build-packages-python-and-ruby.sh
+++ b/build/run-build-packages-python-and-ruby.sh
@@ -52,7 +52,7 @@ gem_wrapper() {
handle_python_package () {
# This function assumes the current working directory is the python package directory
if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
- # This package doesn't need rebuilding.
+ echo "This package doesn't need rebuilding."
return
fi
# Make sure only to use sdist - that's the only format pip can deal with (sigh)
commit 106e69e3bc04d52ac65dfc19463215817a75320e
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Nov 24 10:26:36 2020 -0500
17106: Add examples to S3 auth instructions.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/doc/api/keep-s3.html.textile.liquid b/doc/api/keep-s3.html.textile.liquid
index b1d2c740d..bee91516b 100644
--- a/doc/api/keep-s3.html.textile.liquid
+++ b/doc/api/keep-s3.html.textile.liquid
@@ -74,5 +74,16 @@ h3. Authorization mechanisms
Keep-web accepts AWS Signature Version 4 (AWS4-HMAC-SHA256) as well as the older V2 AWS signature.
-* If your client uses V4 signatures exclusively, and your Arvados token was issued by the same cluster you are connecting to: use the Arvados token's UUID part as AccessKey, and its secret part as SecretKey. This is preferred, where applicable.
-* In all other cases, replace every "/" in your Arvados token with "_", and use the resulting string as both AccessKey and SecretKey.
+If your client uses V4 signatures exclusively _and_ your Arvados token was issued by the same cluster you are connecting to, you can use the Arvados token's UUID part as your S3 Access Key, and its secret part as your S3 Secret Key. This is preferred, where applicable.
+
+Example using cluster @zzzzz@:
+* Arvados token: @v2/zzzzz-gj3su-yyyyyyyyyyyyyyy/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Access Key: @zzzzz-gj3su-yyyyyyyyyyyyyyy@
+* Secret Key: @xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+
+In all other cases, replace every @/@ character in your Arvados token with @_@, and use the resulting string as both Access Key and Secret Key.
+
+Example using a cluster other than @zzzzz@ _or_ an S3 client that uses V2 signatures:
+* Arvados token: @v2/zzzzz-gj3su-yyyyyyyyyyyyyyy/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Access Key: @v2_zzzzz-gj3su-yyyyyyyyyyyyyyy_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Secret Key: @v2_zzzzz-gj3su-yyyyyyyyyyyyyyy_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
commit 5513ae5fa4abed9b3f871173f08d4a0aa52418af
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Nov 23 23:45:47 2020 -0500
17106: Fix key unescape: don't convert + to space.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 02bc19dde..4ee69f277 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -120,7 +120,7 @@ func unescapeKey(key string) string {
// avoid colliding with the Authorization header
// format.
return strings.Replace(key, "_", "/", -1)
- } else if s, err := url.QueryUnescape(key); err == nil {
+ } else if s, err := url.PathUnescape(key); err == nil {
return s
} else {
return key
commit c9e619571b8be5d6576c9a3f74e877bf39ce02c9
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Nov 23 14:52:57 2020 -0500
17106: Allow use of URL-encoded token as S3 access/secret key.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index f85b5592c..02bc19dde 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -16,6 +16,7 @@ import (
"net/url"
"os"
"path/filepath"
+ "regexp"
"sort"
"strconv"
"strings"
@@ -111,6 +112,21 @@ func s3signature(secretKey, scope, signedHeaders, stringToSign string) (string,
return hashdigest(hmac.New(sha256.New, key), stringToSign), nil
}
+var v2tokenUnderscore = regexp.MustCompile(`^v2_[a-z0-9]{5}-gj3su-[a-z0-9]{15}_`)
+
+func unescapeKey(key string) string {
+ if v2tokenUnderscore.MatchString(key) {
+ // Entire Arvados token, with "/" replaced by "_" to
+ // avoid colliding with the Authorization header
+ // format.
+ return strings.Replace(key, "_", "/", -1)
+ } else if s, err := url.QueryUnescape(key); err == nil {
+ return s
+ } else {
+ return key
+ }
+}
+
// checks3signature verifies the given S3 V4 signature and returns the
// Arvados token that corresponds to the given accessKey. An error is
// returned if accessKey is not a valid token UUID or the signature
@@ -152,14 +168,7 @@ func (h *handler) checks3signature(r *http.Request) (string, error) {
} else {
// Access key and secret key are both an entire
// Arvados token or OIDC access token.
- mungedKey := key
- if strings.HasPrefix(key, "v2_") {
- // Entire Arvados token, with "/" replaced by
- // "_" to avoid colliding with the
- // Authorization header format.
- mungedKey = strings.Replace(key, "_", "/", -1)
- }
- ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+mungedKey)
+ ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+unescapeKey(key))
err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/current", nil, nil)
secret = key
}
@@ -190,13 +199,7 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
http.Error(w, "malformed Authorization header", http.StatusUnauthorized)
return true
}
- token = split[0]
- if strings.HasPrefix(token, "v2_") {
- // User provided a full Arvados token with "/"
- // munged to "_" (see V4 signature validation)
- // but client software used S3 V2 signature.
- token = strings.Replace(token, "_", "/", -1)
- }
+ token = unescapeKey(split[0])
} else if strings.HasPrefix(auth, s3SignAlgorithm+" ") {
t, err := h.checks3signature(r)
if err != nil {
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index f8dc60086..bff197ded 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -10,6 +10,7 @@ import (
"fmt"
"io/ioutil"
"net/http"
+ "net/url"
"os"
"os/exec"
"strings"
@@ -118,11 +119,15 @@ func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
secretkey string
}{
{true, aws.V2Signature, arvadostest.ActiveToken, "none"},
+ {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
+ {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
{false, aws.V2Signature, "none", "none"},
{false, aws.V2Signature, "none", arvadostest.ActiveToken},
{true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
{true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
+ {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
+ {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
{false, aws.V4Signature, arvadostest.ActiveToken, ""},
{false, aws.V4Signature, arvadostest.ActiveToken, "none"},
{false, aws.V4Signature, "none", arvadostest.ActiveToken},
commit 51251e7473df5f1a9036d36f2d38d9dd1788f9cc
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Nov 20 10:40:30 2020 -0500
17106: Recommend using full tokens for S3 access.
Accept munged ("/" => "_") tokens in S3 requests with V2 signatures.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/doc/api/keep-s3.html.textile.liquid b/doc/api/keep-s3.html.textile.liquid
index a5a9eeb78..b1d2c740d 100644
--- a/doc/api/keep-s3.html.textile.liquid
+++ b/doc/api/keep-s3.html.textile.liquid
@@ -75,4 +75,4 @@ h3. Authorization mechanisms
Keep-web accepts AWS Signature Version 4 (AWS4-HMAC-SHA256) as well as the older V2 AWS signature.
* If your client uses V4 signatures exclusively, and your Arvados token was issued by the same cluster you are connecting to: use the Arvados token's UUID part as AccessKey, and its secret part as SecretKey. This is preferred, where applicable.
-* If your client uses V2 signatures, or a combination of V2 and V4, or the Arvados token UUID is unknown, or a LoginCluster is in use: use the secret part of the Arvados token for both AccessKey and SecretKey.
+* In all other cases, replace every "/" in your Arvados token with "_", and use the resulting string as both AccessKey and SecretKey.
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 0170146c0..f85b5592c 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -191,6 +191,12 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
return true
}
token = split[0]
+ if strings.HasPrefix(token, "v2_") {
+ // User provided a full Arvados token with "/"
+ // munged to "_" (see V4 signature validation)
+ // but client software used S3 V2 signature.
+ token = strings.Replace(token, "_", "/", -1)
+ }
} else if strings.HasPrefix(auth, s3SignAlgorithm+" ") {
t, err := h.checks3signature(r)
if err != nil {
commit 77adea164640b48948ccd8292f11d9354d2ebd6b
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Nov 18 10:10:02 2020 -0500
17106: Update docs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/doc/api/keep-s3.html.textile.liquid b/doc/api/keep-s3.html.textile.liquid
index d5ad1dc60..a5a9eeb78 100644
--- a/doc/api/keep-s3.html.textile.liquid
+++ b/doc/api/keep-s3.html.textile.liquid
@@ -74,5 +74,5 @@ h3. Authorization mechanisms
Keep-web accepts AWS Signature Version 4 (AWS4-HMAC-SHA256) as well as the older V2 AWS signature.
-* If your client uses V4 signatures exclusively: use the Arvados token's UUID part as AccessKey, and its secret part as SecretKey. This is preferred.
-* If your client uses V2 signatures, or a combination of V2 and V4, or the Arvados token UUID is unknown: use the secret part of the Arvados token for both AccessKey and SecretKey.
+* If your client uses V4 signatures exclusively, and your Arvados token was issued by the same cluster you are connecting to: use the Arvados token's UUID part as AccessKey, and its secret part as SecretKey. This is preferred, where applicable.
+* If your client uses V2 signatures, or a combination of V2 and V4, or the Arvados token UUID is unknown, or a LoginCluster is in use: use the secret part of the Arvados token for both AccessKey and SecretKey.
commit 1944cf47cac3607ab69b31da4adde434662f62c5
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Nov 18 10:02:40 2020 -0500
17106: Comment on stored_secret behavior.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index 1c1c669de..6b308a231 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -345,6 +345,11 @@ class ApiClientAuthorization < ArvadosModel
auth.user = user
auth.api_client_id = 0
end
+ # If stored_secret is set, we save stored_secret in the database
+ # but return the real secret to the caller. This way, if we end
+ # up returning the auth record to the client, they see the same
+ # secret they supplied, instead of the HMAC we saved in the
+ # database.
stored_secret = stored_secret || secret
auth.update_attributes!(user: user,
api_token: stored_secret,
commit 6452fa1e968d0dd33eb2628fec5323bf4f8bbe8c
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Nov 17 15:42:28 2020 -0500
17106: Improve handling of bare tokens issued by remote clusters.
When caching, use the remote cluster's original token UUID.
When returning the current api_client_authorization record, include
the secret supplied by the caller, even when only storing the HMAC in
the database.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index 74a4c1efa..1c1c669de 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -130,6 +130,7 @@ class ApiClientAuthorization < ArvadosModel
token_uuid = ''
secret = token
+ stored_secret = nil # ...if different from secret
optional = nil
case token[0..2]
@@ -206,8 +207,7 @@ class ApiClientAuthorization < ArvadosModel
# below. If so, we'll stuff the database with hmac instead of
# the real OIDC token.
upstream_cluster_id = Rails.configuration.Login.LoginCluster
- token_uuid = upstream_cluster_id + generate_uuid[5..27]
- secret = hmac
+ stored_secret = hmac
else
return nil
end
@@ -246,6 +246,23 @@ class ApiClientAuthorization < ArvadosModel
remote_user_prefix = remote_user['uuid'][0..4]
+ if token_uuid == ''
+ # Use the same UUID as the remote when caching the token.
+ begin
+ remote_token = SafeJSON.load(
+ clnt.get_content('https://' + host + '/arvados/v1/api_client_authorizations/current',
+ {'remote' => Rails.configuration.ClusterID},
+ {'Authorization' => 'Bearer ' + token}))
+ token_uuid = remote_token['uuid']
+ if !token_uuid.match(HasUuid::UUID_REGEX) || token_uuid[0..4] != upstream_cluster_id
+ raise "remote cluster #{upstream_cluster_id} returned invalid token uuid #{token_uuid.inspect}"
+ end
+ rescue => e
+ Rails.logger.warn "error getting remote token details for #{token.inspect}: #{e}"
+ return nil
+ end
+ end
+
# Clusters can only authenticate for their own users.
if remote_user_prefix != upstream_cluster_id
Rails.logger.warn "remote authentication rejected: claimed remote user #{remote_user_prefix} but token was issued by #{upstream_cluster_id}"
@@ -328,11 +345,13 @@ class ApiClientAuthorization < ArvadosModel
auth.user = user
auth.api_client_id = 0
end
+ stored_secret = stored_secret || secret
auth.update_attributes!(user: user,
- api_token: secret,
+ api_token: stored_secret,
api_client_id: 0,
expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh)
- Rails.logger.debug "cached remote token #{token_uuid} with secret #{secret} in local db"
+ Rails.logger.debug "cached remote token #{token_uuid} with secret #{stored_secret} in local db"
+ auth.api_token = secret
return auth
end
commit deaadbea28f273c4528394606a18a9443b30ea02
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Nov 17 15:42:11 2020 -0500
17106: Clean up test.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index d85ca7b96..05ce62e58 100644
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -295,71 +295,63 @@ func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
conn1 := s.conn("z1111")
rootctx1, _, _ := s.rootClients("z1111")
- userctx1, ac1, kc1, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
+ userctx1, ac1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
conn3 := s.conn("z3333")
- _, ac3, kc3 := s.clientsWithToken("z3333", ac1.AuthToken)
-
- // Create a collection on z1111
- var coll arvados.Collection
- fs1, err := coll.FileSystem(ac1, kc1)
- c.Assert(err, check.IsNil)
- f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
- c.Assert(err, check.IsNil)
- _, err = io.WriteString(f, testText)
- c.Assert(err, check.IsNil)
- err = f.Close()
- c.Assert(err, check.IsNil)
- mtxt, err := fs1.MarshalManifest(".")
- c.Assert(err, check.IsNil)
- coll1, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
- "manifest_text": mtxt,
- }})
- c.Assert(err, check.IsNil)
- // Create same collection on z3333
- fs3, err := coll.FileSystem(ac3, kc3)
- c.Assert(err, check.IsNil)
- f, err = fs3.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
- c.Assert(err, check.IsNil)
- _, err = io.WriteString(f, testText)
- c.Assert(err, check.IsNil)
- err = f.Close()
- c.Assert(err, check.IsNil)
- mtxt, err = fs3.MarshalManifest(".")
- c.Assert(err, check.IsNil)
- coll3, err := conn3.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
- "manifest_text": mtxt,
- }})
- c.Assert(err, check.IsNil)
+ createColl := func(clusterID string) arvados.Collection {
+ _, ac, kc := s.clientsWithToken(clusterID, ac1.AuthToken)
+ var coll arvados.Collection
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, testText)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll, err = s.conn(clusterID).CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": mtxt,
+ }})
+ c.Assert(err, check.IsNil)
+ return coll
+ }
for _, trial := range []struct {
- label string
- conn *rpc.Conn
- coll arvados.Collection
+ clusterID string // create the collection on this cluster (then use z3333 to access it)
+ token string
}{
- {"z1111", conn1, coll1},
- {"z3333", conn3, coll3},
+ // Try the hardest test first: z3333 hasn't seen
+ // z1111's token yet, and we're just passing the
+ // opaque secret part, so z3333 has to guess that it
+ // belongs to z1111.
+ {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
+ {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
+ {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
+ {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
} {
- c.Logf("================ %s", trial.label)
- cfgjson, err := trial.conn.ConfigGet(userctx1)
+ c.Logf("================ %v", trial)
+ coll := createColl(trial.clusterID)
+
+ cfgjson, err := conn3.ConfigGet(userctx1)
c.Assert(err, check.IsNil)
var cluster arvados.Cluster
err = json.Unmarshal(cfgjson, &cluster)
c.Assert(err, check.IsNil)
c.Logf("TokenV2 is %s", ac1.AuthToken)
- mungedtoken := strings.Replace(ac1.AuthToken, "/", "_", -1)
host := cluster.Services.WebDAV.ExternalURL.Host
s3args := []string{
"--ssl", "--no-check-certificate",
"--host=" + host, "--host-bucket=" + host,
- "--access_key=" + mungedtoken, "--secret_key=" + mungedtoken,
+ "--access_key=" + trial.token, "--secret_key=" + trial.token,
}
- buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+trial.coll.UUID)...).CombinedOutput()
+ buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
c.Check(err, check.IsNil)
- c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+trial.coll.UUID+`/test.txt\n`)
+ c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
- buf, err = exec.Command("s3cmd", append(s3args, "get", "s3://"+trial.coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
+ buf, err = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
// Command fails because we don't return Etag header.
// c.Check(err, check.IsNil)
flen := strconv.Itoa(len(testText))
commit b6122315820c594ac4d1d5b157dd90523a79946a
Author: Tom Clegg <tom at tomclegg.ca>
Date: Tue Nov 17 09:39:50 2020 -0500
17106: Skip s3cmd test if s3cmd not installed.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index dab5caf01..d85ca7b96 100644
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -286,6 +286,11 @@ func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
}
func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
+ if _, err := exec.LookPath("s3cmd"); err != nil {
+ c.Skip("s3cmd not in PATH")
+ return
+ }
+
testText := "IntegrationSuite.TestS3WithFederatedToken"
conn1 := s.conn("z1111")
commit 2325be1cf9fc9cd34ae826688347cf7a9c01e9aa
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Nov 16 21:25:12 2020 -0500
17106: Test S3 with modified v2 token issued by LoginCluster.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index 03b760eaa..dab5caf01 100644
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -9,6 +9,7 @@ import (
"context"
"database/sql"
"encoding/json"
+ "fmt"
"io"
"io/ioutil"
"math"
@@ -16,7 +17,10 @@ import (
"net/http"
"net/url"
"os"
+ "os/exec"
"path/filepath"
+ "strconv"
+ "strings"
"git.arvados.org/arvados.git/lib/boot"
"git.arvados.org/arvados.git/lib/config"
@@ -281,6 +285,83 @@ func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
c.Check(coll.PortableDataHash, check.Equals, pdh)
}
+func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
+ testText := "IntegrationSuite.TestS3WithFederatedToken"
+
+ conn1 := s.conn("z1111")
+ rootctx1, _, _ := s.rootClients("z1111")
+ userctx1, ac1, kc1, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
+ conn3 := s.conn("z3333")
+ _, ac3, kc3 := s.clientsWithToken("z3333", ac1.AuthToken)
+
+ // Create a collection on z1111
+ var coll arvados.Collection
+ fs1, err := coll.FileSystem(ac1, kc1)
+ c.Assert(err, check.IsNil)
+ f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, testText)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ mtxt, err := fs1.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll1, err := conn1.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": mtxt,
+ }})
+ c.Assert(err, check.IsNil)
+
+ // Create same collection on z3333
+ fs3, err := coll.FileSystem(ac3, kc3)
+ c.Assert(err, check.IsNil)
+ f, err = fs3.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, testText)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ mtxt, err = fs3.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll3, err := conn3.CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": mtxt,
+ }})
+ c.Assert(err, check.IsNil)
+
+ for _, trial := range []struct {
+ label string
+ conn *rpc.Conn
+ coll arvados.Collection
+ }{
+ {"z1111", conn1, coll1},
+ {"z3333", conn3, coll3},
+ } {
+ c.Logf("================ %s", trial.label)
+ cfgjson, err := trial.conn.ConfigGet(userctx1)
+ c.Assert(err, check.IsNil)
+ var cluster arvados.Cluster
+ err = json.Unmarshal(cfgjson, &cluster)
+ c.Assert(err, check.IsNil)
+
+ c.Logf("TokenV2 is %s", ac1.AuthToken)
+ mungedtoken := strings.Replace(ac1.AuthToken, "/", "_", -1)
+ host := cluster.Services.WebDAV.ExternalURL.Host
+ s3args := []string{
+ "--ssl", "--no-check-certificate",
+ "--host=" + host, "--host-bucket=" + host,
+ "--access_key=" + mungedtoken, "--secret_key=" + mungedtoken,
+ }
+ buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+trial.coll.UUID)...).CombinedOutput()
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+trial.coll.UUID+`/test.txt\n`)
+
+ buf, err = exec.Command("s3cmd", append(s3args, "get", "s3://"+trial.coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
+ // Command fails because we don't return Etag header.
+ // c.Check(err, check.IsNil)
+ flen := strconv.Itoa(len(testText))
+ c.Check(string(buf), check.Matches, `(?ms).*`+flen+` of `+flen+`.*`)
+ }
+}
+
func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
conn1 := s.conn("z1111")
conn3 := s.conn("z3333")
commit b377684bf4c7a6211e39556c744544857ee66493
Author: Tom Clegg <tom at tomclegg.ca>
Date: Mon Nov 16 20:37:05 2020 -0500
17106: Accept v2 token with / replaced by _ as s3 access/secret key.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 57c9d7efb..0170146c0 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -152,7 +152,14 @@ func (h *handler) checks3signature(r *http.Request) (string, error) {
} else {
// Access key and secret key are both an entire
// Arvados token or OIDC access token.
- ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+key)
+ mungedKey := key
+ if strings.HasPrefix(key, "v2_") {
+ // Entire Arvados token, with "/" replaced by
+ // "_" to avoid colliding with the
+ // Authorization header format.
+ mungedKey = strings.Replace(key, "_", "/", -1)
+ }
+ ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+mungedKey)
err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/current", nil, nil)
secret = key
}
@@ -170,7 +177,7 @@ func (h *handler) checks3signature(r *http.Request) (string, error) {
} else if expect != signature {
return "", fmt.Errorf("signature does not match (scope %q signedHeaders %q stringToSign %q)", scope, signedHeaders, stringToSign)
}
- return secret, nil
+ return aca.TokenV2(), nil
}
// serveS3 handles r and returns true if r is a request from an S3
commit a31656998dc52d163c6be7c40918daa1d5f16048
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 23 14:48:35 2020 -0500
Fix OS package iteration number in run-build-docker-jobs-image.sh
refs #17012
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh
index 94a05671a..075771821 100755
--- a/build/run-build-docker-jobs-image.sh
+++ b/build/run-build-docker-jobs-image.sh
@@ -143,14 +143,13 @@ fi
# package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
-echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
-
-if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
- python_sdk_version="${python_sdk_version}-1"
-else
- python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
+if [[ -z "$cwl_runner_version" ]]; then
+ echo "ERROR: cwl_runner_version is empty";
+ exit 1
fi
+echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
+
# For development and release candidate packages, the OS package has a "~dev"
# or "~rc" suffix, but Python requires a ".dev" or "rc" suffix.
#
@@ -163,15 +162,16 @@ fi
python_sdk_version_os=$(echo -n $python_sdk_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
cwl_runner_version_os=$(echo -n $cwl_runner_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
-if [[ -z "$cwl_runner_version" ]]; then
- echo "ERROR: cwl_runner_version is empty";
- exit 1
+if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
+ python_sdk_version_os="${python_sdk_version_os}-1"
+else
+ python_sdk_version_os="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
fi
-if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
- cwl_runner_version="${cwl_runner_version}-1"
+if [[ "${cwl_runner_version_os}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
+ cwl_runner_version_os="${cwl_runner_version_os}-1"
else
- cwl_runner_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
+ cwl_runner_version_os="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
fi
cd docker/jobs
commit f7499eb0eeb8f9cbfad1545f7e196f8939fb0f05
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 23 11:51:21 2020 -0500
Update error checking in run-build-docker-jobs-image.sh
refs #17012
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh
index d83af708a..94a05671a 100755
--- a/build/run-build-docker-jobs-image.sh
+++ b/build/run-build-docker-jobs-image.sh
@@ -163,8 +163,8 @@ fi
python_sdk_version_os=$(echo -n $python_sdk_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
cwl_runner_version_os=$(echo -n $cwl_runner_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
-if [[ -z "$cwl_runner_version_tag" ]]; then
- echo "ERROR: cwl_runner_version_tag is empty";
+if [[ -z "$cwl_runner_version" ]]; then
+ echo "ERROR: cwl_runner_version is empty";
exit 1
fi
commit d5d15b19d43ba894912d710e9baa1538d5f29ef3
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 23 11:47:53 2020 -0500
Remove now-unneeded code from sdk/cwl/test_with_arvbox.sh, add comment
to clarify that calculate_python_sdk_cwl_package_versions now outputs
python-style package suffixes.
refs #17012
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index 935bec63b..0021bc8d9 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -141,9 +141,11 @@ else
. /usr/src/arvados/build/run-library.sh
TMPHERE=\$(pwd)
cd /usr/src/arvados
+
+ # This defines python_sdk_version and cwl_runner_version with python-style
+ # package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
- cwl_runner_version=\$(echo -n \$cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
cd \$TMPHERE
set -u
commit d577b0e4f583d5aa4cb01af81ee0b326d9f81f29
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Nov 23 10:59:06 2020 -0500
17154: Add comment. Use strings.HasPrefix
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/apps/workbench/app/controllers/users_controller.rb b/apps/workbench/app/controllers/users_controller.rb
index d782bcb40..21ea7a8e6 100644
--- a/apps/workbench/app/controllers/users_controller.rb
+++ b/apps/workbench/app/controllers/users_controller.rb
@@ -39,6 +39,17 @@ class UsersController < ApplicationController
def profile
params[:offer_return_to] ||= params[:return_to]
+
+ # In a federation situation, when you get a user record using
+ # "current user of token" it can fetch a stale user record from
+ # the local cluster. So even if profile settings were just written
+ # to the user record on the login cluster (because the user just
+ # filled out the profile), those profile settings may not appear
+ # in the "current user" response because it is returning a cached
+ # record from the local cluster.
+ #
+ # In this case, explicitly fetching user record forces it to get a
+ # fresh record from the login cluster.
Thread.current[:user] = User.find(current_user.uuid)
end
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index 5571e89e9..247556e33 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -530,7 +530,7 @@ func (conn *Conn) UserUpdate(ctx context.Context, options arvados.UpdateOptions)
if err != nil {
return resp, err
}
- if options.UUID[:5] != conn.cluster.ClusterID {
+ if !strings.HasPrefix(options.UUID, conn.cluster.ClusterID) {
// Copy the updated user record to the local cluster
err = conn.batchUpdateUsers(ctx, arvados.ListOptions{}, []arvados.User{resp})
if err != nil {
commit 7995f95d0989b3974c9f1d39da4f36c74ae958cf
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Nov 20 16:01:25 2020 -0500
17154: Copy updates on federated users to local cluster
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index 3ec17c17e..5571e89e9 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -526,7 +526,18 @@ func (conn *Conn) UserUpdate(ctx context.Context, options arvados.UpdateOptions)
if options.BypassFederation {
return conn.local.UserUpdate(ctx, options)
}
- return conn.chooseBackend(options.UUID).UserUpdate(ctx, options)
+ resp, err := conn.chooseBackend(options.UUID).UserUpdate(ctx, options)
+ if err != nil {
+ return resp, err
+ }
+ if options.UUID[:5] != conn.cluster.ClusterID {
+ // Copy the updated user record to the local cluster
+ err = conn.batchUpdateUsers(ctx, arvados.ListOptions{}, []arvados.User{resp})
+ if err != nil {
+ return arvados.User{}, err
+ }
+ }
+ return resp, err
}
func (conn *Conn) UserUpdateUUID(ctx context.Context, options arvados.UpdateUUIDOptions) (arvados.User, error) {
commit a15943fc33acd69fb4b706fb97c10635a4a9dbb5
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Nov 20 15:25:27 2020 -0500
17154: Make sure most current user record is loaded.
Convert 'option' items (which are symbols) to strings for value comparison
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/apps/workbench/app/controllers/application_controller.rb b/apps/workbench/app/controllers/application_controller.rb
index cf4bfa8c5..6d139cd5f 100644
--- a/apps/workbench/app/controllers/application_controller.rb
+++ b/apps/workbench/app/controllers/application_controller.rb
@@ -760,7 +760,7 @@ class ApplicationController < ActionController::Base
if current_user && !profile_config.empty?
current_user_profile = current_user.prefs[:profile]
profile_config.each do |k, entry|
- if entry['Required']
+ if entry[:Required]
if !current_user_profile ||
!current_user_profile[k] ||
current_user_profile[k].empty?
diff --git a/apps/workbench/app/controllers/users_controller.rb b/apps/workbench/app/controllers/users_controller.rb
index 27fc12bf4..d782bcb40 100644
--- a/apps/workbench/app/controllers/users_controller.rb
+++ b/apps/workbench/app/controllers/users_controller.rb
@@ -39,6 +39,7 @@ class UsersController < ApplicationController
def profile
params[:offer_return_to] ||= params[:return_to]
+ Thread.current[:user] = User.find(current_user.uuid)
end
def activity
diff --git a/apps/workbench/app/views/users/profile.html.erb b/apps/workbench/app/views/users/profile.html.erb
index 6692196da..caa22bda1 100644
--- a/apps/workbench/app/views/users/profile.html.erb
+++ b/apps/workbench/app/views/users/profile.html.erb
@@ -68,29 +68,30 @@ SPDX-License-Identifier: AGPL-3.0 %>
</div>
<% profile_config.kind_of?(Array) && profile_config.andand.each do |entry| %>
- <% if entry['Key'] %>
+ <% if entry[:Key] %>
<%
show_save_button = true
- label = entry['Required'] ? '* ' : ''
- label += entry['FormFieldTitle']
- value = current_user_profile[entry['Key'].to_sym] if current_user_profile
+ label = entry[:Required] ? '* ' : ''
+ label += entry[:FormFieldTitle]
+ value = current_user_profile[entry[:Key].to_sym] if current_user_profile
%>
<div class="form-group">
- <label for="<%=entry['Key']%>"
+ <label for="<%=entry[:Key]%>"
class="col-sm-3 control-label"
- style=<%="color:red" if entry['Required']&&(!value||value.empty?)%>> <%=label%>
+ style=<%="color:red" if entry[:Required]&&(!value||value.empty?)%>> <%=label%>
</label>
- <% if entry['Type'] == 'select' %>
+ <% if entry[:Type] == 'select' %>
<div class="col-sm-8">
- <select class="form-control" name="user[prefs][profile][<%=entry['Key']%>]">
- <% entry['Options'].each do |option, _| %>
+ <select class="form-control" name="user[prefs][profile][<%=entry[:Key]%>]">
+ <% entry[:Options].each do |option, _| %>
+ <% option = option.to_s %>
<option value="<%=option%>" <%='selected' if option==value%>><%=option%></option>
<% end %>
</select>
</div>
<% else %>
<div class="col-sm-8">
- <input type="text" class="form-control" name="user[prefs][profile][<%=entry['Key']%>]" placeholder="<%=entry['FormFieldDescription']%>" value="<%=value%>" ></input>
+ <input type="text" class="form-control" name="user[prefs][profile][<%=entry[:Key]%>]" placeholder="<%=entry[:FormFieldDescription]%>" value="<%=value%>" ></input>
</div>
<% end %>
</div>
commit 568168423fa929cbfa05d3c6bca591017ba00e44
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 23 09:20:56 2020 -0500
Now that calculate_python_sdk_cwl_package_versions returns python-style
version string suffixes (.dev/rc), make sure the 2 scripts that use that
function are adapted accordingly.
refs #17012
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index 0e570d5f3..af838d68e 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -69,10 +69,10 @@ fi
. build/run-library.sh
+# This defines python_sdk_version and cwl_runner_version with python-style
+# package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
-cwl_runner_version=$(echo -n $cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
-
set -x
docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
echo arv-keepdocker arvados/jobs $cwl_runner_version
diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh
index 59914a2ee..d83af708a 100755
--- a/build/run-build-docker-jobs-image.sh
+++ b/build/run-build-docker-jobs-image.sh
@@ -139,6 +139,8 @@ if [[ -z "$ARVADOS_BUILDING_VERSION" ]] && ! [[ -z "$version_tag" ]]; then
ARVADOS_BUILDING_ITERATION="1"
fi
+# This defines python_sdk_version and cwl_runner_version with python-style
+# package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
@@ -149,13 +151,17 @@ else
python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
fi
-# What we use to tag the Docker image. For development and release
-# candidate packages, the OS package has a "~dev" or "~rc" suffix, but
-# Python requires a ".dev" or "rc" suffix. Arvados-cwl-runner will be
-# expecting the Python-compatible version string when it tries to pull
-# the Docker image, but --build-arg is expecting the OS package
+# For development and release candidate packages, the OS package has a "~dev"
+# or "~rc" suffix, but Python requires a ".dev" or "rc" suffix.
+#
+# Arvados-cwl-runner will be expecting the Python-compatible version string
+# when it tries to pull the Docker image, so we use that to tag the Docker
+# image.
+#
+# The --build-arg docker invocation arguments are expecting the OS package
# version.
-cwl_runner_version_tag=$(echo -n $cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
+python_sdk_version_os=$(echo -n $python_sdk_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
+cwl_runner_version_os=$(echo -n $cwl_runner_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
if [[ -z "$cwl_runner_version_tag" ]]; then
echo "ERROR: cwl_runner_version_tag is empty";
@@ -170,10 +176,10 @@ fi
cd docker/jobs
docker build $NOCACHE \
- --build-arg python_sdk_version=${python_sdk_version} \
- --build-arg cwl_runner_version=${cwl_runner_version} \
+ --build-arg python_sdk_version=${python_sdk_version_os} \
+ --build-arg cwl_runner_version=${cwl_runner_version_os} \
--build-arg repo_version=${REPO} \
- -t arvados/jobs:$cwl_runner_version_tag .
+ -t arvados/jobs:$cwl_runner_version .
ECODE=$?
@@ -207,7 +213,7 @@ else
## 20150526 nico -- *sometimes* dockerhub needs re-login
## even though credentials are already in .dockercfg
docker login -u arvados
- docker_push arvados/jobs:$cwl_runner_version_tag
+ docker_push arvados/jobs:$cwl_runner_version
title "upload arvados images finished (`timer`)"
else
title "upload arvados images SKIPPED because no --upload option set (`timer`)"
commit 076d6df6431377b1af8ca94491e3ef7c1f21052a
Author: Ward Vandewege <ward at curii.com>
Date: Sun Nov 22 08:35:06 2020 -0500
Move the handle_python_package() function from run-library.sh to
run-build-packages-python-and-ruby.sh, it is only used in the latter.
Update it to call python3 instead of just 'python'.
refs #15888
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/run-build-packages-python-and-ruby.sh b/build/run-build-packages-python-and-ruby.sh
index f3b7564d7..84da280d7 100755
--- a/build/run-build-packages-python-and-ruby.sh
+++ b/build/run-build-packages-python-and-ruby.sh
@@ -6,7 +6,6 @@
COLUMNS=80
. `dirname "$(readlink -f "$0")"`/run-library.sh
-#. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
read -rd "\000" helpmessage <<EOF
$(basename $0): Build Arvados Python packages and Ruby gems
@@ -50,6 +49,16 @@ gem_wrapper() {
title "End of $gem_name gem build (`timer`)"
}
+handle_python_package () {
+ # This function assumes the current working directory is the python package directory
+ if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
+ # This package doesn't need rebuilding.
+ return
+ fi
+ # Make sure only to use sdist - that's the only format pip can deal with (sigh)
+ python3 setup.py $DASHQ_UNLESS_DEBUG sdist
+}
+
python_wrapper() {
local package_name="$1"; shift
local package_directory="$1"; shift
diff --git a/build/run-library.sh b/build/run-library.sh
index a72aa3ad7..6f95a8f4b 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -79,16 +79,6 @@ calculate_python_sdk_cwl_package_versions() {
cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
}
-handle_python_package () {
- # This function assumes the current working directory is the python package directory
- if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
- # This package doesn't need rebuilding.
- return
- fi
- # Make sure only to use sdist - that's the only format pip can deal with (sigh)
- python setup.py $DASHQ_UNLESS_DEBUG sdist
-}
-
handle_ruby_gem() {
local gem_name="$1"; shift
local gem_version="$(nohash_version_from_git)"
commit 4ecea81f99d1ba947e1b0bb1eaed22e6d88dabe5
Author: Ward Vandewege <ward at curii.com>
Date: Sat Nov 21 15:53:01 2020 -0500
Fix more golint warnings.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/controller/rpc/conn_test.go b/lib/controller/rpc/conn_test.go
index f43cc1dde..cf4dbc476 100644
--- a/lib/controller/rpc/conn_test.go
+++ b/lib/controller/rpc/conn_test.go
@@ -24,7 +24,11 @@ func Test(t *testing.T) {
var _ = check.Suite(&RPCSuite{})
-const contextKeyTestTokens = "testTokens"
+type key int
+
+const (
+ contextKeyTestTokens key = iota
+)
type RPCSuite struct {
log logrus.FieldLogger
commit e6c3e65fb6c5d80ba4364665b96cc3d9a7f3ac00
Author: Ward Vandewege <ward at curii.com>
Date: Fri Nov 20 17:31:50 2020 -0500
Remove one more python2 remnant. This is a fix for the
python3-arvados-cwl-runner package, it now installs /usr/bin/cwltool
again.
refs #15888
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/run-library.sh b/build/run-library.sh
index 1716cf370..a72aa3ad7 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -690,9 +690,9 @@ fpm_build_virtualenv () {
done
fi
- # the python-arvados-cwl-runner package comes with cwltool, expose that version
- if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
- COMMAND_ARR+=("usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
+ # the python3-arvados-cwl-runner package comes with cwltool, expose that version
+ if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
+ COMMAND_ARR+=("usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
fi
COMMAND_ARR+=(".")
commit ff711779ea897ba208e3c6880e53fa98db484e51
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Nov 19 15:36:29 2020 -0500
17015: Ensure that containers belong to this cluster
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 4115482d8..a5899ce8a 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -202,7 +202,7 @@ var containerUuidPattern = regexp.MustCompile(`^[a-z0-9]{5}-dz642-[a-z0-9]{15}$`
// Cancelled or Complete. See https://dev.arvados.org/issues/10979
func (disp *Dispatcher) checkSqueueForOrphans() {
for _, uuid := range disp.sqCheck.All() {
- if !containerUuidPattern.MatchString(uuid) {
+ if !containerUuidPattern.MatchString(uuid) || !strings.HasPrefix(uuid, disp.cluster.ClusterID) {
continue
}
err := disp.TrackContainer(uuid)
commit 456fc0518fba438448c34d233a8c68371aa6bffe
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu Nov 19 16:48:15 2020 -0500
17009: Mention S3 considerations in keep-web install doc.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/doc/install/install-keep-web.html.textile.liquid b/doc/install/install-keep-web.html.textile.liquid
index 24f37bfb4..b797c1958 100644
--- a/doc/install/install-keep-web.html.textile.liquid
+++ b/doc/install/install-keep-web.html.textile.liquid
@@ -20,7 +20,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
h2(#introduction). Introduction
-The Keep-web server provides read/write HTTP (WebDAV) access to files stored in Keep. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
+The Keep-web server provides read/write access to files stored in Keep using WebDAV and S3 protocols. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
h2(#dns). Configure DNS
@@ -61,6 +61,8 @@ Collections can be served from their own subdomain:
</code></pre>
</notextile>
+This option is preferred if you plan to access Keep using third-party S3 client software, because it accommodates S3 virtual host-style requests and path-style requests without any special client configuration.
+
h4. Under the main domain
Alternately, they can go under the main domain by including @--@:
commit 047dab2b789c6db8cb43624dd284f703f8903268
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Nov 18 17:35:29 2020 -0500
17009: Support accessing S3 with virtual hosted-style URLs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/doc/api/keep-s3.html.textile.liquid b/doc/api/keep-s3.html.textile.liquid
index 2cae81761..d5ad1dc60 100644
--- a/doc/api/keep-s3.html.textile.liquid
+++ b/doc/api/keep-s3.html.textile.liquid
@@ -21,7 +21,11 @@ To access Arvados S3 using an S3 client library, you must tell it to use the URL
The "bucket name" is an Arvados collection uuid, portable data hash, or project uuid.
-The bucket name must be encoded as the first path segment of every request. This is what the S3 documentation calls "Path-Style Requests".
+Path-style and virtual host-style requests are supported.
+* A path-style request uses the hostname indicated by @Services.WebDAVDownload.ExternalURL@, with the bucket name in the first path segment: @https://download.example.com/zzzzz-4zz18-asdfgasdfgasdfg/@.
+* A virtual host-style request uses the hostname pattern indicated by @Services.WebDAV.ExternalURL@, with a bucket name in place of the leading @*@: @https://zzzzz-4zz18-asdfgasdfgasdfg.collections.example.com/@.
+
+If you have wildcard DNS, TLS, and routing set up, an S3 client configured with endpoint @collections.example.com@ should work regardless of which request style it uses.
h3. Supported Operations
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 49fb2456f..57c9d7efb 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -205,7 +205,15 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
fs := client.SiteFileSystem(kc)
fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
- objectNameGiven := strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+ var objectNameGiven bool
+ fspath := "/by_id"
+ if id := parseCollectionIDFromDNSName(r.Host); id != "" {
+ fspath += "/" + id
+ objectNameGiven = true
+ } else {
+ objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+ }
+ fspath += r.URL.Path
switch {
case r.Method == http.MethodGet && !objectNameGiven:
@@ -221,7 +229,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
}
return true
case r.Method == http.MethodGet || r.Method == http.MethodHead:
- fspath := "/by_id" + r.URL.Path
fi, err := fs.Stat(fspath)
if r.Method == "HEAD" && !objectNameGiven {
// HeadBucket
@@ -255,7 +262,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
http.Error(w, "missing object name in PUT request", http.StatusBadRequest)
return true
}
- fspath := "by_id" + r.URL.Path
var objectIsDir bool
if strings.HasSuffix(fspath, "/") {
if !h.Config.cluster.Collections.S3FolderObjects {
@@ -350,7 +356,6 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
http.Error(w, "missing object name in DELETE request", http.StatusBadRequest)
return true
}
- fspath := "by_id" + r.URL.Path
if strings.HasSuffix(fspath, "/") {
fspath = strings.TrimSuffix(fspath, "/")
fi, err := fs.Stat(fspath)
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index 786e68afe..f8dc60086 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -700,3 +700,12 @@ func (s *IntegrationSuite) TestS3cmd(c *check.C) {
c.Check(err, check.IsNil)
c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
}
+
+func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+
+ hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
+ c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
+ c.Check(body, check.Equals, "⛵\n")
+}
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index acdc11b30..43817b51f 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -257,12 +257,16 @@ func (s *IntegrationSuite) Test200(c *check.C) {
}
// Return header block and body.
-func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
+func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
curlArgs := []string{"--silent", "--show-error", "--include"}
testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
- if token != "" {
- curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
+ if strings.Contains(auth, " ") {
+ // caller supplied entire Authorization header value
+ curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
+ } else if auth != "" {
+ // caller supplied Arvados token
+ curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
}
curlArgs = append(curlArgs, args...)
curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
commit 9e0399b7fcff3ea83ee986b07f60b8b27659d5c9
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 16 20:40:47 2020 -0500
Fix more golint warnings.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index ec56e902c..21913ff96 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -227,7 +227,7 @@ func (kc *KeepClient) getOrHead(method string, locator string, header http.Heade
var retryList []string
for triesRemaining > 0 {
- triesRemaining -= 1
+ triesRemaining--
retryList = nil
for _, host := range serversToTry {
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 8d595fbe1..59c412724 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -209,7 +209,7 @@ func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.
fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id"))
if fh.count == 0 {
resp.WriteHeader(500)
- fh.count += 1
+ fh.count++
fh.handled <- fmt.Sprintf("http://%s", req.Host)
} else {
fh.successhandler.ServeHTTP(resp, req)
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 6a31c98bc..91117f2d3 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -161,7 +161,7 @@ func (this *KeepClient) putReplicas(
lastError := make(map[string]string)
for retriesRemaining > 0 {
- retriesRemaining -= 1
+ retriesRemaining--
nextServer = 0
retryServers = []string{}
for replicasTodo > 0 {
@@ -170,8 +170,8 @@ func (this *KeepClient) putReplicas(
if nextServer < len(sv) {
DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[nextServer])
go this.uploadToKeepServer(sv[nextServer], hash, getReader(), uploadStatusChan, expectedLength, reqid)
- nextServer += 1
- active += 1
+ nextServer++
+ active++
} else {
if active == 0 && retriesRemaining == 0 {
msg := "Could not write sufficient replicas: "
@@ -190,7 +190,7 @@ func (this *KeepClient) putReplicas(
// Now wait for something to happen.
if active > 0 {
status := <-uploadStatusChan
- active -= 1
+ active--
if status.statusCode == 200 {
// good news!
commit 8f2b8255ab3e58911b56942530e322e0fcd954e2
Author: Ward Vandewege <ward at curii.com>
Date: Tue Nov 17 17:35:31 2020 -0500
17012: switch calculate_python_sdk_cwl_package_versions() to use the
python version calculation code. This effectively is bash calling
python calling bash, which I don't like, but the code is simpler.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh
index d2c9f7274..59914a2ee 100755
--- a/build/run-build-docker-jobs-image.sh
+++ b/build/run-build-docker-jobs-image.sh
@@ -157,6 +157,11 @@ fi
# version.
cwl_runner_version_tag=$(echo -n $cwl_runner_version | sed s/~dev/.dev/g | sed s/~rc/rc/g)
+if [[ -z "$cwl_runner_version_tag" ]]; then
+ echo "ERROR: cwl_runner_version_tag is empty";
+ exit 1
+fi
+
if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
cwl_runner_version="${cwl_runner_version}-1"
else
diff --git a/build/run-library.sh b/build/run-library.sh
index c925cf1cd..1716cf370 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -75,25 +75,8 @@ timestamp_from_git() {
}
calculate_python_sdk_cwl_package_versions() {
- python_sdk_ts=$(cd sdk/python && timestamp_from_git)
- cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
- build_ts=$(cd build && timestamp_from_git version-at-commit.sh)
-
- ar=($python_sdk_ts $cwl_runner_ts $build_ts)
- OLDIFS=$IFS
- IFS=$'\n'
- max=`echo "${ar[*]}" | sort -nr | head -n1`
- IFS=$OLDIFS
-
- python_sdk_version=$(cd sdk/python && nohash_version_from_git)
- cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git)
- build_version=$(cd build && nohash_version_from_git version-at-commit.sh)
-
- if [[ $max -eq $build_ts ]]; then
- cwl_runner_version=$build_version
- elif [[ $max -eq $python_sdk_ts ]]; then
- cwl_runner_version=$python_sdk_version
- fi
+ python_sdk_version=$(cd sdk/python && python3 arvados_version.py)
+ cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
}
handle_python_package () {
diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index 006194935..c3936617f 100644
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -6,6 +6,7 @@ import subprocess
import time
import os
import re
+import sys
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
VERSION_PATHS = {
@@ -23,7 +24,7 @@ def choose_version_from():
sorted_ts = sorted(ts.items())
getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR)
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
return getver
def git_version_at_commit():
@@ -51,7 +52,11 @@ def get_version(setup_dir, module):
try:
save_version(setup_dir, module, git_version_at_commit())
except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err))
+ print("ERROR: {0}".format(err), file=sys.stderr)
pass
return read_version(setup_dir, module)
+
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version(SETUP_DIR, "arvados_cwl"))
diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index 154a56607..092131d93 100644
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -6,6 +6,7 @@ import subprocess
import time
import os
import re
+import sys
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
VERSION_PATHS = {
@@ -22,7 +23,7 @@ def choose_version_from():
sorted_ts = sorted(ts.items())
getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR)
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
return getver
def git_version_at_commit():
@@ -50,7 +51,11 @@ def get_version(setup_dir, module):
try:
save_version(setup_dir, module, git_version_at_commit())
except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err))
+ print("ERROR: {0}".format(err), file=sys.stderr)
pass
return read_version(setup_dir, module)
+
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version(SETUP_DIR, "arvados"))
diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index 154a56607..38e6f564e 100644
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -6,6 +6,7 @@ import subprocess
import time
import os
import re
+import sys
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
VERSION_PATHS = {
@@ -22,7 +23,7 @@ def choose_version_from():
sorted_ts = sorted(ts.items())
getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR)
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
return getver
def git_version_at_commit():
@@ -50,7 +51,7 @@ def get_version(setup_dir, module):
try:
save_version(setup_dir, module, git_version_at_commit())
except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err))
+ print("ERROR: {0}".format(err), file=sys.stderr)
pass
return read_version(setup_dir, module)
diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index 02b22e8ea..d8eec3d9e 100644
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -6,6 +6,7 @@ import subprocess
import time
import os
import re
+import sys
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
VERSION_PATHS = {
@@ -23,7 +24,7 @@ def choose_version_from():
sorted_ts = sorted(ts.items())
getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR)
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
return getver
def git_version_at_commit():
@@ -51,7 +52,7 @@ def get_version(setup_dir, module):
try:
save_version(setup_dir, module, git_version_at_commit())
except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err))
+ print("ERROR: {0}".format(err), file=sys.stderr)
pass
return read_version(setup_dir, module)
diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index 02b22e8ea..d8eec3d9e 100644
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -6,6 +6,7 @@ import subprocess
import time
import os
import re
+import sys
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
VERSION_PATHS = {
@@ -23,7 +24,7 @@ def choose_version_from():
sorted_ts = sorted(ts.items())
getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR)
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
return getver
def git_version_at_commit():
@@ -51,7 +52,7 @@ def get_version(setup_dir, module):
try:
save_version(setup_dir, module, git_version_at_commit())
except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err))
+ print("ERROR: {0}".format(err), file=sys.stderr)
pass
return read_version(setup_dir, module)
commit fff3ea11c8658d1363e5d2da4a4a7df3055b8f76
Author: Ward Vandewege <ward at curii.com>
Date: Tue Nov 17 10:26:58 2020 -0500
17012: update calculation of python version number in run-library.sh
(used by a few docker image scripts).
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/run-library.sh b/build/run-library.sh
index c598270ce..c925cf1cd 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -61,11 +61,12 @@ version_from_git() {
}
nohash_version_from_git() {
+ local subdir="$1"; shift
if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
echo "$ARVADOS_BUILDING_VERSION"
return
fi
- version_from_git | cut -d. -f1-4
+ version_from_git $subdir | cut -d. -f1-4
}
timestamp_from_git() {
@@ -76,11 +77,21 @@ timestamp_from_git() {
calculate_python_sdk_cwl_package_versions() {
python_sdk_ts=$(cd sdk/python && timestamp_from_git)
cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
+ build_ts=$(cd build && timestamp_from_git version-at-commit.sh)
+
+ ar=($python_sdk_ts $cwl_runner_ts $build_ts)
+ OLDIFS=$IFS
+ IFS=$'\n'
+ max=`echo "${ar[*]}" | sort -nr | head -n1`
+ IFS=$OLDIFS
python_sdk_version=$(cd sdk/python && nohash_version_from_git)
cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git)
+ build_version=$(cd build && nohash_version_from_git version-at-commit.sh)
- if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
+ if [[ $max -eq $build_ts ]]; then
+ cwl_runner_version=$build_version
+ elif [[ $max -eq $python_sdk_ts ]]; then
cwl_runner_version=$python_sdk_version
fi
}
commit 3dc9a7d7830c84f5c235c706f473066589a5c6cd
Author: Ward Vandewege <ward at curii.com>
Date: Tue Nov 17 09:52:53 2020 -0500
17012: remove old gittaggers.py files.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py
deleted file mode 100644
index d6a4c24a7..000000000
--- a/sdk/cwl/gittaggers.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from builtins import str
-from builtins import next
-
-from setuptools.command.egg_info import egg_info
-import subprocess
-import time
-import os
-
-SETUP_DIR = os.path.dirname(__file__) or '.'
-
-def choose_version_from():
- sdk_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
- cwl_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', SETUP_DIR]).strip()
- if int(sdk_ts) > int(cwl_ts):
- getver = os.path.join(SETUP_DIR, "../python")
- else:
- getver = SETUP_DIR
- return getver
-
-class EggInfoFromGit(egg_info):
- """Tag the build with git commit timestamp.
-
- If a build tag has already been set (e.g., "egg_info -b", building
- from source package), leave it alone.
- """
- def git_latest_tag(self):
- gittags = subprocess.check_output(['git', 'tag', '-l']).split()
- gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
- return str(next(iter(gittags)).decode('utf-8'))
-
- def git_timestamp_tag(self):
- gitinfo = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', choose_version_from()]).strip()
- return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
-
- def tags(self):
- if self.tag_build is None:
- self.tag_build = self.git_latest_tag() + self.git_timestamp_tag()
- return egg_info.tags(self)
diff --git a/sdk/python/gittaggers.py b/sdk/python/gittaggers.py
deleted file mode 100644
index f3278fcc1..000000000
--- a/sdk/python/gittaggers.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from setuptools.command.egg_info import egg_info
-import subprocess
-import time
-
-class EggInfoFromGit(egg_info):
- """Tag the build with git commit timestamp.
-
- If a build tag has already been set (e.g., "egg_info -b", building
- from source package), leave it alone.
- """
- def git_latest_tag(self):
- gittags = subprocess.check_output(['git', 'tag', '-l']).split()
- gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
- return str(next(iter(gittags)).decode('utf-8'))
-
- def git_timestamp_tag(self):
- gitinfo = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', '.']).strip()
- return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
-
- def tags(self):
- if self.tag_build is None:
- self.tag_build = self.git_latest_tag()+self.git_timestamp_tag()
- return egg_info.tags(self)
diff --git a/services/dockercleaner/gittaggers.py b/services/dockercleaner/gittaggers.py
deleted file mode 120000
index a9ad861d8..000000000
--- a/services/dockercleaner/gittaggers.py
+++ /dev/null
@@ -1 +0,0 @@
-../../sdk/python/gittaggers.py
\ No newline at end of file
diff --git a/services/fuse/gittaggers.py b/services/fuse/gittaggers.py
deleted file mode 120000
index a9ad861d8..000000000
--- a/services/fuse/gittaggers.py
+++ /dev/null
@@ -1 +0,0 @@
-../../sdk/python/gittaggers.py
\ No newline at end of file
diff --git a/tools/crunchstat-summary/gittaggers.py b/tools/crunchstat-summary/gittaggers.py
deleted file mode 120000
index a9ad861d8..000000000
--- a/tools/crunchstat-summary/gittaggers.py
+++ /dev/null
@@ -1 +0,0 @@
-../../sdk/python/gittaggers.py
\ No newline at end of file
commit 8ffbca22ca9f6172fe52a5ae2de40a6b7d537978
Author: Ward Vandewege <ward at curii.com>
Date: Mon Nov 16 15:08:26 2020 -0500
17012: when calculating the version of our Python packages, take the
build directory into account.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index d5f48c066..006194935 100644
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -8,25 +8,29 @@ import os
import re
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../python")),
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
def choose_version_from():
- sdk_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
- cwl_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', SETUP_DIR]).strip()
- if int(sdk_ts) > int(cwl_ts):
- getver = os.path.join(SETUP_DIR, "../python")
- else:
- getver = SETUP_DIR
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR)
return getver
def git_version_at_commit():
curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
- myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
return myversion
def save_version(setup_dir, module, v):
@@ -46,7 +50,8 @@ def get_version(setup_dir, module):
else:
try:
save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError):
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err))
pass
return read_version(setup_dir, module)
diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index 36804bf5b..154a56607 100644
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -7,11 +7,29 @@ import time
import os
import re
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
+
+def choose_version_from():
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR)
+ return getver
+
def git_version_at_commit():
- curdir = os.path.dirname(os.path.abspath(__file__))
+ curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
- myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
return myversion
def save_version(setup_dir, module, v):
@@ -31,7 +49,8 @@ def get_version(setup_dir, module):
else:
try:
save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError):
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err))
pass
return read_version(setup_dir, module)
diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index 36804bf5b..154a56607 100644
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -7,11 +7,29 @@ import time
import os
import re
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
+
+def choose_version_from():
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR)
+ return getver
+
def git_version_at_commit():
- curdir = os.path.dirname(os.path.abspath(__file__))
+ curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
- myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
return myversion
def save_version(setup_dir, module, v):
@@ -31,7 +49,8 @@ def get_version(setup_dir, module):
else:
try:
save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError):
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err))
pass
return read_version(setup_dir, module)
diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index 0d307c1be..02b22e8ea 100644
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -8,25 +8,29 @@ import os
import re
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
def choose_version_from():
- sdk_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip()
- cwl_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', SETUP_DIR]).strip()
- if int(sdk_ts) > int(cwl_ts):
- getver = os.path.join(SETUP_DIR, "../../sdk/python")
- else:
- getver = SETUP_DIR
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR)
return getver
def git_version_at_commit():
curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
- myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
return myversion
def save_version(setup_dir, module, v):
@@ -46,7 +50,8 @@ def get_version(setup_dir, module):
else:
try:
save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError):
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err))
pass
return read_version(setup_dir, module)
diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index 0d307c1be..02b22e8ea 100644
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -8,25 +8,29 @@ import os
import re
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
def choose_version_from():
- sdk_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip()
- cwl_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', SETUP_DIR]).strip()
- if int(sdk_ts) > int(cwl_ts):
- getver = os.path.join(SETUP_DIR, "../../sdk/python")
- else:
- getver = SETUP_DIR
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR)
return getver
def git_version_at_commit():
curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
- myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
return myversion
def save_version(setup_dir, module, v):
@@ -46,7 +50,8 @@ def get_version(setup_dir, module):
else:
try:
save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError):
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err))
pass
return read_version(setup_dir, module)
commit 48a6fe360bd9f5ad59c7e5114898b98a2413e898
Author: Ward Vandewege <ward at curii.com>
Date: Wed Nov 18 16:20:40 2020 -0500
16950: switch to os.UserHomeDir(), and send all output to stderr.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/cmd.go b/lib/costanalyzer/cmd.go
index 800860ddf..9b0685225 100644
--- a/lib/costanalyzer/cmd.go
+++ b/lib/costanalyzer/cmd.go
@@ -33,7 +33,6 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
}()
logger.SetFormatter(new(NoPrefixFormatter))
- logger.SetOutput(stdout)
loader := config.NewLoader(stdin, logger)
loader.SkipLegacy = true
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index ddd7abc63..4284542b8 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -17,7 +17,6 @@ import (
"io/ioutil"
"net/http"
"os"
- "os/user"
"strconv"
"strings"
"time"
@@ -227,12 +226,12 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str
if !cache {
reload = true
} else {
- user, err := user.Current()
+ homeDir, err := os.UserHomeDir()
if err != nil {
reload = true
- logger.Info("Unable to determine current user, not using cache")
+ logger.Info("Unable to determine current user home directory, not using cache")
} else {
- cacheDir = user.HomeDir + "/.cache/arvados/costanalyzer/"
+ cacheDir = homeDir + "/.cache/arvados/costanalyzer/"
err = ensureDirectory(logger, cacheDir)
if err != nil {
reload = true
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 862f5161c..4fab93bf4 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -163,13 +163,13 @@ func (*Suite) TestContainerRequestUUID(c *check.C) {
// Run costanalyzer with 1 container request uuid
exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
@@ -182,7 +182,7 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
// Run costanalyzer with 2 container request uuids
exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
@@ -193,7 +193,7 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
@@ -222,7 +222,7 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
// Run costanalyzer with the project uuid
exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID, "-cache=false", "-log-level", "debug", "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
uuidReport, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
@@ -233,7 +233,7 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
re = regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches = re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches = re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err = ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
@@ -246,7 +246,7 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
// Run costanalyzer with 2 container request uuids
exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
c.Assert(err, check.IsNil)
@@ -257,7 +257,7 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
commit a06f784c9f8d2acb2ec9baa852193117b8c1cfdd
Author: Ward Vandewege <ward at curii.com>
Date: Wed Nov 18 10:09:51 2020 -0500
16950: make specifying of the output directory mandatory (remove
default).
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index 8f5395f58..ddd7abc63 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -92,8 +92,8 @@ Options:
flags.PrintDefaults()
}
loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
- resultsDir = *flags.String("output", "results", "output `directory` for the CSV reports")
- flags.Var(&uuids, "uuid", "Toplevel `project or container request` uuid. May be specified more than once.")
+ flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports (required)")
+ flags.Var(&uuids, "uuid", "Toplevel `project or container request` uuid. May be specified more than once. (required)")
flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
err = flags.Parse(args)
if err == flag.ErrHelp {
@@ -112,6 +112,13 @@ Options:
return
}
+ if resultsDir == "" {
+ flags.Usage()
+ err = fmt.Errorf("Error: output directory must be specified")
+ exitCode = 2
+ return
+ }
+
lvl, err := logrus.ParseLevel(*loglevel)
if err != nil {
exitCode = 2
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 8253121c9..862f5161c 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -161,7 +161,7 @@ func (*Suite) TestUsage(c *check.C) {
func (*Suite) TestContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 1 container request uuid
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
@@ -180,7 +180,7 @@ func (*Suite) TestContainerRequestUUID(c *check.C) {
func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 2 container request uuids
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
@@ -220,7 +220,7 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
c.Assert(err, check.IsNil)
// Run costanalyzer with the project uuid
- exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID, "-cache=false", "-log-level", "debug"}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID, "-cache=false", "-log-level", "debug", "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
@@ -244,7 +244,7 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 2 container request uuids
- exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID, "-output", "results"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
commit 3e1c102325332298de9a2d0de74026e846a0d9af
Author: Ward Vandewege <ward at curii.com>
Date: Sat Nov 14 13:22:50 2020 -0500
16950: refactor caching; use ~/.cache/arvados/costanalyzer for that
purpose.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index 319303b92..8f5395f58 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -17,6 +17,7 @@ import (
"io/ioutil"
"net/http"
"os"
+ "os/user"
"strconv"
"strings"
"time"
@@ -117,6 +118,9 @@ Options:
return
}
logger.SetLevel(lvl)
+ if !cache {
+ logger.Debug("Caching disabled\n")
+ }
return
}
@@ -170,6 +174,10 @@ func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.Container
func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
reload = true
+ if strings.Contains(uuid, "-j7d0g-") {
+ // We do not cache projects, they have no final state
+ return
+ }
// See if we have a cached copy of this object
_, err := os.Stat(file)
if err != nil {
@@ -188,15 +196,13 @@ func loadCachedObject(logger *logrus.Logger, file string, uuid string, object in
// See if it is in a final state, if that makes sense
switch v := object.(type) {
- case arvados.Group:
- // Projects (j7d0g) do not have state so they should always be reloaded
- case arvados.Container:
- if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled {
+ case *arvados.ContainerRequest:
+ if v.State == arvados.ContainerRequestStateFinal {
reload = false
logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
}
- case arvados.ContainerRequest:
- if v.State == arvados.ContainerRequestStateFinal {
+ case *arvados.Container:
+ if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled {
reload = false
logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
}
@@ -206,18 +212,28 @@ func loadCachedObject(logger *logrus.Logger, file string, uuid string, object in
// Load an Arvados object.
func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid string, cache bool, object interface{}) (err error) {
- err = ensureDirectory(logger, path)
- if err != nil {
- return
- }
-
- file := path + "/" + uuid + ".json"
+ file := uuid + ".json"
var reload bool
+ var cacheDir string
+
if !cache {
reload = true
} else {
- reload = loadCachedObject(logger, file, uuid, &object)
+ user, err := user.Current()
+ if err != nil {
+ reload = true
+ logger.Info("Unable to determine current user, not using cache")
+ } else {
+ cacheDir = user.HomeDir + "/.cache/arvados/costanalyzer/"
+ err = ensureDirectory(logger, cacheDir)
+ if err != nil {
+ reload = true
+ logger.Infof("Unable to create cache directory at %s, not using cache: %s", cacheDir, err.Error())
+ } else {
+ reload = loadCachedObject(logger, cacheDir+file, uuid, object)
+ }
+ }
}
if !reload {
return
@@ -242,10 +258,12 @@ func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid str
err = fmt.Errorf("error marshaling object with UUID %q:\n %s", uuid, err)
return
}
- err = ioutil.WriteFile(file, encoded, 0644)
- if err != nil {
- err = fmt.Errorf("error writing file %s:\n %s", file, err)
- return
+ if cacheDir != "" {
+ err = ioutil.WriteFile(cacheDir+file, encoded, 0644)
+ if err != nil {
+ err = fmt.Errorf("error writing file %s:\n %s", file, err)
+ return
+ }
}
return
}
@@ -289,7 +307,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
cost = make(map[string]float64)
var project arvados.Group
- err = loadObject(logger, ac, resultsDir+"/"+uuid, uuid, cache, &project)
+ err = loadObject(logger, ac, uuid, uuid, cache, &project)
if err != nil {
return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error())
}
@@ -344,13 +362,12 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
// This is a container request, find the container
var cr arvados.ContainerRequest
- err = loadObject(logger, ac, resultsDir+"/"+uuid, uuid, cache, &cr)
+ err = loadObject(logger, ac, uuid, uuid, cache, &cr)
if err != nil {
return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
}
- fmt.Printf("cr: %+v\n", cr)
var container arvados.Container
- err = loadObject(logger, ac, resultsDir+"/"+uuid, cr.ContainerUUID, cache, &container)
+ err = loadObject(logger, ac, uuid, cr.ContainerUUID, cache, &container)
if err != nil {
return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
}
@@ -388,7 +405,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
}
logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
var c2 arvados.Container
- err = loadObject(logger, ac, resultsDir+"/"+uuid, cr2.ContainerUUID, cache, &c2)
+ err = loadObject(logger, ac, uuid, cr2.ContainerUUID, cache, &c2)
if err != nil {
return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
}
@@ -407,6 +424,7 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
if err != nil {
return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
}
+ logger.Infof("\nUUID report in %s\n\n", fName)
return
}
@@ -471,11 +489,6 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
}
}
- logger.Info("\n")
- for k := range cost {
- logger.Infof("Uuid report in %s/%s.csv\n", resultsDir, k)
- }
-
if len(cost) == 0 {
logger.Info("Nothing to do!\n")
return
@@ -504,6 +517,6 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
exitcode = 1
return
}
- logger.Infof("\nAggregate cost accounting for all supplied uuids in %s\n", aFile)
+ logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
return
}
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index e2fb9e99b..8253121c9 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -245,8 +245,6 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 2 container request uuids
exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
- c.Logf("%s", stderr.Bytes())
- c.Logf("%s", stdout.Bytes())
c.Check(exitcode, check.Equals, 0)
c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
commit f8c45afb252a2b6620a14594c874bca3217fb6e3
Author: Ward Vandewege <ward at curii.com>
Date: Fri Nov 13 16:22:57 2020 -0500
16950: more changes after review.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/cmd.go b/lib/costanalyzer/cmd.go
index 6829be7d3..800860ddf 100644
--- a/lib/costanalyzer/cmd.go
+++ b/lib/costanalyzer/cmd.go
@@ -28,7 +28,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
logger := ctxlog.New(stderr, "text", "info")
defer func() {
if err != nil {
- logger.WithError(err).Error("fatal")
+ logger.Error("\n" + err.Error() + "\n")
}
}()
@@ -38,7 +38,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
loader := config.NewLoader(stdin, logger)
loader.SkipLegacy = true
- exitcode := costanalyzer(prog, args, loader, logger, stdout, stderr)
+ exitcode, err := costanalyzer(prog, args, loader, logger, stdout, stderr)
return exitcode
}
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index ccd9520bc..319303b92 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -5,8 +5,6 @@
package costanalyzer
import (
- "bytes"
- "context"
"encoding/json"
"errors"
"flag"
@@ -26,52 +24,17 @@ import (
"github.com/sirupsen/logrus"
)
-// LegacyNodeInfo is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3)
-// Example:
-// {
-// "total_cpu_cores":2,
-// "total_scratch_mb":33770,
-// "cloud_node":
-// {
-// "price":0.1,
-// "size":"m4.large"
-// },
-// "total_ram_mb":7986
-// }
-type LegacyNodeInfo struct {
- CPUCores int64 `json:"total_cpu_cores"`
- ScratchMb int64 `json:"total_scratch_mb"`
- RAMMb int64 `json:"total_ram_mb"`
- CloudNode LegacyCloudNode `json:"cloud_node"`
-}
-
-// LegacyCloudNode is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3)
-type LegacyCloudNode struct {
- Price float64 `json:"price"`
- Size string `json:"size"`
-}
-
-// Node is a struct for records created by Arvados Dispatch Cloud (Arvados >= 2.0.0)
-// Example:
-// {
-// "Name": "Standard_D1_v2",
-// "ProviderType": "Standard_D1_v2",
-// "VCPUs": 1,
-// "RAM": 3584000000,
-// "Scratch": 50000000000,
-// "IncludedScratch": 50000000000,
-// "AddedScratch": 0,
-// "Price": 0.057,
-// "Preemptible": false
-//}
-type Node struct {
- VCPUs int64
- Scratch int64
- RAM int64
- Price float64
- Name string
+type nodeInfo struct {
+ // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
+ Properties struct {
+ CloudNode struct {
+ Price float64
+ Size string
+ } `json:"cloud_node"`
+ }
+ // Modern
ProviderType string
- Preemptible bool
+ Price float64
}
type arrayFlags []string
@@ -85,7 +48,7 @@ func (i *arrayFlags) Set(value string) error {
return nil
}
-func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool) {
+func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) {
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.SetOutput(stderr)
flags.Usage = func() {
@@ -127,12 +90,13 @@ Options:
`, prog)
flags.PrintDefaults()
}
- loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)")
- resultsDir = *flags.String("output", "results", "output directory for the CSV reports")
- flags.Var(&uuids, "uuid", "Toplevel project or container request uuid. May be specified more than once.")
+ loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
+ resultsDir = *flags.String("output", "results", "output `directory` for the CSV reports")
+ flags.Var(&uuids, "uuid", "Toplevel `project or container request` uuid. May be specified more than once.")
flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
- err := flags.Parse(args)
+ err = flags.Parse(args)
if err == flag.ErrHelp {
+ err = nil
exitCode = 1
return
} else if err != nil {
@@ -141,8 +105,8 @@ Options:
}
if len(uuids) < 1 {
- logger.Errorf("Error: no uuid(s) provided")
flags.Usage()
+ err = fmt.Errorf("Error: no uuid(s) provided")
exitCode = 2
return
}
@@ -161,90 +125,87 @@ func ensureDirectory(logger *logrus.Logger, dir string) (err error) {
if os.IsNotExist(err) {
err = os.MkdirAll(dir, 0700)
if err != nil {
- return fmt.Errorf("Error creating directory %s: %s\n", dir, err.Error())
+ return fmt.Errorf("error creating directory %s: %s", dir, err.Error())
}
} else {
if !statData.IsDir() {
- return fmt.Errorf("The path %s is not a directory\n", dir)
+ return fmt.Errorf("the path %s is not a directory", dir)
}
}
return
}
-func addContainerLine(logger *logrus.Logger, node interface{}, cr, container map[string]interface{}) (csv string, cost float64) {
- csv = cr["uuid"].(string) + ","
- csv += cr["name"].(string) + ","
- csv += container["uuid"].(string) + ","
- csv += container["state"].(string) + ","
- if container["started_at"] != nil {
- csv += container["started_at"].(string) + ","
+func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.ContainerRequest, container arvados.Container) (csv string, cost float64) {
+ csv = cr.UUID + ","
+ csv += cr.Name + ","
+ csv += container.UUID + ","
+ csv += string(container.State) + ","
+ if container.StartedAt != nil {
+ csv += container.StartedAt.String() + ","
} else {
csv += ","
}
var delta time.Duration
- if container["finished_at"] != nil {
- csv += container["finished_at"].(string) + ","
- finishedTimestamp, err := time.Parse("2006-01-02T15:04:05.000000000Z", container["finished_at"].(string))
- if err != nil {
- fmt.Println(err)
- }
- startedTimestamp, err := time.Parse("2006-01-02T15:04:05.000000000Z", container["started_at"].(string))
- if err != nil {
- fmt.Println(err)
- }
- delta = finishedTimestamp.Sub(startedTimestamp)
+ if container.FinishedAt != nil {
+ csv += container.FinishedAt.String() + ","
+ delta = container.FinishedAt.Sub(*container.StartedAt)
csv += strconv.FormatFloat(delta.Seconds(), 'f', 0, 64) + ","
} else {
csv += ",,"
}
var price float64
var size string
- switch n := node.(type) {
- case Node:
- price = n.Price
- size = n.ProviderType
- case LegacyNodeInfo:
- price = n.CloudNode.Price
- size = n.CloudNode.Size
- default:
- logger.Warn("WARNING: unknown node type found!")
+ if node.Properties.CloudNode.Price != 0 {
+ price = node.Properties.CloudNode.Price
+ size = node.Properties.CloudNode.Size
+ } else {
+ price = node.Price
+ size = node.ProviderType
}
cost = delta.Seconds() / 3600 * price
csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
return
}
-func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload bool, object map[string]interface{}) {
+func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
reload = true
// See if we have a cached copy of this object
- if _, err := os.Stat(file); err == nil {
- data, err := ioutil.ReadFile(file)
- if err != nil {
- logger.Errorf("error reading %q: %s", file, err)
- return
- }
- err = json.Unmarshal(data, &object)
- if err != nil {
- logger.Errorf("failed to unmarshal json: %s: %s", data, err)
- return
- }
+ _, err := os.Stat(file)
+ if err != nil {
+ return
+ }
+ data, err := ioutil.ReadFile(file)
+ if err != nil {
+ logger.Errorf("error reading %q: %s", file, err)
+ return
+ }
+ err = json.Unmarshal(data, &object)
+ if err != nil {
+ logger.Errorf("failed to unmarshal json: %s: %s", data, err)
+ return
+ }
- // See if it is in a final state, if that makes sense
+ // See if it is in a final state, if that makes sense
+ switch v := object.(type) {
+ case arvados.Group:
// Projects (j7d0g) do not have state so they should always be reloaded
- if !strings.Contains(uuid, "-j7d0g-") {
- if object["state"].(string) == "Complete" || object["state"].(string) == "Failed" {
- reload = false
- logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
- return
- }
+ case arvados.Container:
+ if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled {
+ reload = false
+ logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
+ }
+ case arvados.ContainerRequest:
+ if v.State == arvados.ContainerRequestStateFinal {
+ reload = false
+ logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
}
}
return
}
// Load an Arvados object.
-func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string, cache bool) (object map[string]interface{}, err error) {
+func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid string, cache bool, object interface{}) (err error) {
err = ensureDirectory(logger, path)
if err != nil {
return
@@ -256,105 +217,70 @@ func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path st
if !cache {
reload = true
} else {
- reload, object = loadCachedObject(logger, file, uuid)
+ reload = loadCachedObject(logger, file, uuid, &object)
}
if !reload {
return
}
if strings.Contains(uuid, "-j7d0g-") {
- err = arv.Get("groups", uuid, nil, &object)
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/groups/"+uuid, nil, nil)
} else if strings.Contains(uuid, "-xvhdp-") {
- err = arv.Get("container_requests", uuid, nil, &object)
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)
} else if strings.Contains(uuid, "-dz642-") {
- err = arv.Get("containers", uuid, nil, &object)
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)
} else {
- err = arv.Get("jobs", uuid, nil, &object)
+ err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err)
+ return
}
if err != nil {
- err = fmt.Errorf("Error loading object with UUID %q:\n %s\n", uuid, err)
+ err = fmt.Errorf("error loading object with UUID %q:\n %s", uuid, err)
return
}
encoded, err := json.MarshalIndent(object, "", " ")
if err != nil {
- err = fmt.Errorf("Error marshaling object with UUID %q:\n %s\n", uuid, err)
+ err = fmt.Errorf("error marshaling object with UUID %q:\n %s", uuid, err)
return
}
err = ioutil.WriteFile(file, encoded, 0644)
if err != nil {
- err = fmt.Errorf("Error writing file %s:\n %s\n", file, err)
+ err = fmt.Errorf("error writing file %s:\n %s", file, err)
return
}
return
}
-func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, itemMap map[string]interface{}) (node interface{}, err error) {
- logUuid, ok := itemMap["log_uuid"]
- if !ok {
+func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
+ if cr.LogUUID == "" {
err = errors.New("No log collection")
return
}
var collection arvados.Collection
- err = arv.Get("collections", logUuid.(string), nil, &collection)
+ err = ac.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
if err != nil {
- err = fmt.Errorf("Error getting collection: %s", err)
+ err = fmt.Errorf("error getting collection: %s", err)
return
}
var fs arvados.CollectionFileSystem
fs, err = collection.FileSystem(ac, kc)
if err != nil {
- err = fmt.Errorf("Error opening collection as filesystem: %s", err)
+ err = fmt.Errorf("error opening collection as filesystem: %s", err)
return
}
var f http.File
f, err = fs.Open("node.json")
if err != nil {
- err = fmt.Errorf("Error opening file 'node.json' in collection %s: %s", logUuid.(string), err)
+ err = fmt.Errorf("error opening file 'node.json' in collection %s: %s", cr.LogUUID, err)
return
}
- var nodeDict map[string]interface{}
- buf := new(bytes.Buffer)
- _, err = buf.ReadFrom(f)
+ err = json.NewDecoder(f).Decode(&node)
if err != nil {
- err = fmt.Errorf("Error reading file 'node.json' in collection %s: %s", logUuid.(string), err)
+ err = fmt.Errorf("error reading file 'node.json' in collection %s: %s", cr.LogUUID, err)
return
}
- contents := buf.String()
- f.Close()
-
- err = json.Unmarshal([]byte(contents), &nodeDict)
- if err != nil {
- err = fmt.Errorf("Error unmarshalling: %s", err)
- return
- }
- if val, ok := nodeDict["properties"]; ok {
- var encoded []byte
- encoded, err = json.MarshalIndent(val, "", " ")
- if err != nil {
- err = fmt.Errorf("Error marshalling: %s", err)
- return
- }
- // node is type LegacyNodeInfo
- var newNode LegacyNodeInfo
- err = json.Unmarshal(encoded, &newNode)
- if err != nil {
- err = fmt.Errorf("Error unmarshalling: %s", err)
- return
- }
- node = newNode
- } else {
- // node is type Node
- var newNode Node
- err = json.Unmarshal([]byte(contents), &newNode)
- if err != nil {
- err = fmt.Errorf("Error unmarshalling: %s", err)
- return
- }
- node = newNode
- }
return
}
@@ -362,20 +288,18 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
cost = make(map[string]float64)
- project, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid, cache)
+ var project arvados.Group
+ err = loadObject(logger, ac, resultsDir+"/"+uuid, uuid, cache, &project)
if err != nil {
- return nil, fmt.Errorf("Error loading object %s: %s\n", uuid, err.Error())
+ return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error())
}
- // arv -f uuid container_request list --filters '[["owner_uuid","=","<someuuid>"],["requesting_container_uuid","=",null]]'
-
- // Now find all container requests that have the container we found above as requesting_container_uuid
var childCrs map[string]interface{}
filterset := []arvados.Filter{
{
Attr: "owner_uuid",
Operator: "=",
- Operand: project["uuid"].(string),
+ Operand: project.UUID,
},
{
Attr: "requesting_container_uuid",
@@ -383,12 +307,12 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
Operand: nil,
},
}
- err = ac.RequestAndDecodeContext(context.Background(), &childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
+ err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
"filters": filterset,
"limit": 10000,
})
if err != nil {
- return nil, fmt.Errorf("Error querying container_requests: %s\n", err.Error())
+ return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
if value, ok := childCrs["items"]; ok {
logger.Infof("Collecting top level container requests in project %s\n", uuid)
@@ -397,7 +321,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
itemMap := item.(map[string]interface{})
crCsv, err := generateCrCsv(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache)
if err != nil {
- return nil, fmt.Errorf("Error generating container_request CSV: %s\n", err.Error())
+ return nil, fmt.Errorf("error generating container_request CSV: %s", err.Error())
}
for k, v := range crCsv {
cost[k] = v
@@ -419,59 +343,59 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
var totalCost float64
// This is a container request, find the container
- cr, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid, cache)
+ var cr arvados.ContainerRequest
+ err = loadObject(logger, ac, resultsDir+"/"+uuid, uuid, cache, &cr)
if err != nil {
- return nil, fmt.Errorf("Error loading object %s: %s", uuid, err)
+ return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
}
- container, err := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string), cache)
+ fmt.Printf("cr: %+v\n", cr)
+ var container arvados.Container
+ err = loadObject(logger, ac, resultsDir+"/"+uuid, cr.ContainerUUID, cache, &container)
if err != nil {
- return nil, fmt.Errorf("Error loading object %s: %s", cr["container_uuid"].(string), err)
+ return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
}
topNode, err := getNode(arv, ac, kc, cr)
if err != nil {
- return nil, fmt.Errorf("Error getting node %s: %s\n", cr["uuid"], err)
+ return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err)
}
tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
csv += tmpCsv
totalCost += tmpTotalCost
- cost[container["uuid"].(string)] = totalCost
+ cost[container.UUID] = totalCost
- // Now find all container requests that have the container we found above as requesting_container_uuid
- var childCrs map[string]interface{}
+ // Find all container requests that have the container we found above as requesting_container_uuid
+ var childCrs arvados.ContainerRequestList
filterset := []arvados.Filter{
{
Attr: "requesting_container_uuid",
Operator: "=",
- Operand: container["uuid"].(string),
+ Operand: container.UUID,
}}
- err = ac.RequestAndDecodeContext(context.Background(), &childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
+ err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
"filters": filterset,
"limit": 10000,
})
if err != nil {
return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
- if value, ok := childCrs["items"]; ok {
- logger.Infof("Collecting child containers for container request %s", uuid)
- items := value.([]interface{})
- for _, item := range items {
- logger.Info(".")
- itemMap := item.(map[string]interface{})
- node, err := getNode(arv, ac, kc, itemMap)
- if err != nil {
- return nil, fmt.Errorf("Error getting node %s: %s\n", itemMap["uuid"], err)
- }
- logger.Debug("\nChild container: " + itemMap["container_uuid"].(string) + "\n")
- c2, err := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string), cache)
- if err != nil {
- return nil, fmt.Errorf("Error loading object %s: %s", cr["container_uuid"].(string), err)
- }
- tmpCsv, tmpTotalCost = addContainerLine(logger, node, itemMap, c2)
- cost[itemMap["container_uuid"].(string)] = tmpTotalCost
- csv += tmpCsv
- totalCost += tmpTotalCost
+ logger.Infof("Collecting child containers for container request %s", uuid)
+ for _, cr2 := range childCrs.Items {
+ logger.Info(".")
+ node, err := getNode(arv, ac, kc, cr2)
+ if err != nil {
+ return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
}
+ logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
+ var c2 arvados.Container
+ err = loadObject(logger, ac, resultsDir+"/"+uuid, cr2.ContainerUUID, cache, &c2)
+ if err != nil {
+ return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
+ }
+ tmpCsv, tmpTotalCost = addContainerLine(logger, node, cr2, c2)
+ cost[cr2.ContainerUUID] = tmpTotalCost
+ csv += tmpCsv
+ totalCost += tmpTotalCost
}
logger.Info(" done\n")
@@ -481,20 +405,19 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
fName := resultsDir + "/" + uuid + ".csv"
err = ioutil.WriteFile(fName, []byte(csv), 0644)
if err != nil {
- return nil, fmt.Errorf("Error writing file with path %s: %s\n", fName, err.Error())
+ return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
}
return
}
-func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int) {
- exitcode, uuids, resultsDir, cache := parseFlags(prog, args, loader, logger, stderr)
+func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
+ exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr)
if exitcode != 0 {
return
}
- err := ensureDirectory(logger, resultsDir)
+ err = ensureDirectory(logger, resultsDir)
if err != nil {
- logger.Errorf("%s", err)
exitcode = 3
return
}
@@ -502,13 +425,13 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
// Arvados Client setup
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
- logger.Errorf("error creating Arvados object: %s", err)
+ err = fmt.Errorf("error creating Arvados object: %s", err)
exitcode = 1
return
}
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
- logger.Errorf("error creating Keep object: %s", err)
+ err = fmt.Errorf("error creating Keep object: %s", err)
exitcode = 1
return
}
@@ -521,8 +444,6 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
// This is a project (group)
cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
if err != nil {
- // FIXME print error
- logger.Info(err.Error())
exitcode = 1
return
}
@@ -531,9 +452,12 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
}
} else if strings.Contains(uuid, "-xvhdp-") {
// This is a container request
- crCsv, err := generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
+ var crCsv map[string]float64
+ crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
if err != nil {
- logger.Fatalf("Error generating container_request CSV: %s\n", err.Error())
+ err = fmt.Errorf("Error generating container_request CSV for uuid %s: %s", uuid, err.Error())
+ exitcode = 2
+ return
}
for k, v := range crCsv {
cost[k] = v
@@ -541,7 +465,8 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
} else if strings.Contains(uuid, "-tpzed-") {
// This is a user. The "Home" project for a user is not a real project.
// It is identified by the user uuid. As such, cost analysis for the
- // "Home" project is not supported by this program.
+ // "Home" project is not supported by this program. Skip this uuid, but
+ // keep going.
logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
}
}
@@ -575,11 +500,10 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
err = ioutil.WriteFile(aFile, []byte(csv), 0644)
if err != nil {
- logger.Errorf("Error writing file with path %s: %s\n", aFile, err.Error())
+ err = fmt.Errorf("Error writing file with path %s: %s", aFile, err.Error())
exitcode = 1
return
- } else {
- logger.Infof("\nAggregate cost accounting for all supplied uuids in %s\n", aFile)
}
+ logger.Infof("\nAggregate cost accounting for all supplied uuids in %s\n", aFile)
return
}
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 0a44be8d8..e2fb9e99b 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -6,7 +6,6 @@ package costanalyzer
import (
"bytes"
- "context"
"io"
"io/ioutil"
"os"
@@ -92,6 +91,18 @@ func (s *Suite) SetUpSuite(c *check.C) {
"Preemptible": false
}`
+ legacyD1V2JSON := `{
+ "properties": {
+ "cloud_node": {
+ "price": 0.073001,
+ "size": "Standard_D1_v2"
+ },
+ "total_cpu_cores": 1,
+ "total_ram_mb": 3418,
+ "total_scratch_mb": 51170
+ }
+}`
+
// Our fixtures do not actually contain file contents. Populate the log collections we're going to use with the node.json file
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID, arvadostest.LogCollectionUUID, standardE4sV3JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID2, arvadostest.LogCollectionUUID2, standardD32sV3JSON)
@@ -100,19 +111,19 @@ func (s *Suite) SetUpSuite(c *check.C) {
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest2UUID, arvadostest.DiagnosticsContainerRequest2LogCollectionUUID, standardA1V2JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher1ContainerRequestUUID, arvadostest.Hasher1LogCollectionUUID, standardA1V2JSON)
createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher2ContainerRequestUUID, arvadostest.Hasher2LogCollectionUUID, standardA2V2JSON)
- createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, legacyD1V2JSON)
}
func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) {
// Get the CR
var cr arvados.ContainerRequest
- err := ac.RequestAndDecodeContext(context.Background(), &cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil)
+ err := ac.RequestAndDecode(&cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil)
c.Assert(err, check.Equals, nil)
c.Assert(cr.LogUUID, check.Equals, logUUID)
// Get the log collection
var coll arvados.Collection
- err = ac.RequestAndDecodeContext(context.Background(), &coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
+ err = ac.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
c.Assert(err, check.IsNil)
// Create a node.json file -- the fixture doesn't actually contain the contents of the collection.
@@ -131,7 +142,7 @@ func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Cl
c.Assert(mtxt, check.NotNil)
// Update collection record
- err = ac.RequestAndDecodeContext(context.Background(), &coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{
+ err = ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{
"collection": map[string]interface{}{
"manifest_text": mtxt,
},
@@ -195,13 +206,13 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
// the analysis with the project uuid. The results should be identical.
ac := arvados.NewClientFromEnv()
var cr arvados.ContainerRequest
- err = ac.RequestAndDecodeContext(context.Background(), &cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{
+ err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{
"container_request": map[string]interface{}{
"owner_uuid": arvadostest.AProjectUUID,
},
})
c.Assert(err, check.IsNil)
- err = ac.RequestAndDecodeContext(context.Background(), &cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{
+ err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{
"container_request": map[string]interface{}{
"owner_uuid": arvadostest.AProjectUUID,
},
@@ -234,16 +245,18 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
var stdout, stderr bytes.Buffer
// Run costanalyzer with 2 container request uuids
exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Logf("%s", stderr.Bytes())
+ c.Logf("%s", stdout.Bytes())
c.Check(exitcode, check.Equals, 0)
c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00914539")
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192")
uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
c.Assert(err, check.IsNil)
- c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00586435")
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
@@ -251,5 +264,5 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
- c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01490377")
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030")
}
commit 04df85b3ace293a3c1ad59e7ad279f53dbdf14fa
Author: Ward Vandewege <ward at curii.com>
Date: Wed Nov 11 22:07:48 2020 -0500
16950: changes after review.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/command.go b/lib/costanalyzer/cmd.go
similarity index 97%
rename from lib/costanalyzer/command.go
rename to lib/costanalyzer/cmd.go
index 0760b4fd2..6829be7d3 100644
--- a/lib/costanalyzer/command.go
+++ b/lib/costanalyzer/cmd.go
@@ -33,6 +33,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
}()
logger.SetFormatter(new(NoPrefixFormatter))
+ logger.SetOutput(stdout)
loader := config.NewLoader(stdin, logger)
loader.SkipLegacy = true
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index c86e26769..ccd9520bc 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -6,6 +6,7 @@ package costanalyzer
import (
"bytes"
+ "context"
"encoding/json"
"errors"
"flag"
@@ -16,7 +17,6 @@ import (
"git.arvados.org/arvados.git/sdk/go/keepclient"
"io"
"io/ioutil"
- "log"
"net/http"
"os"
"strconv"
@@ -26,9 +26,6 @@ import (
"github.com/sirupsen/logrus"
)
-// Dict is a helper type so we don't have to write out 'map[string]interface{}' every time.
-type Dict map[string]interface{}
-
// LegacyNodeInfo is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3)
// Example:
// {
@@ -88,7 +85,7 @@ func (i *arrayFlags) Set(value string) error {
return nil
}
-func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string) {
+func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool) {
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.SetOutput(stderr)
flags.Usage = func() {
@@ -133,6 +130,7 @@ Options:
loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)")
resultsDir = *flags.String("output", "results", "output directory for the CSV reports")
flags.Var(&uuids, "uuid", "Toplevel project or container request uuid. May be specified more than once.")
+ flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
err := flags.Parse(args)
if err == flag.ErrHelp {
exitCode = 1
@@ -163,19 +161,17 @@ func ensureDirectory(logger *logrus.Logger, dir string) (err error) {
if os.IsNotExist(err) {
err = os.MkdirAll(dir, 0700)
if err != nil {
- logger.Errorf("Error creating directory %s: %s\n", dir, err.Error())
- return
+ return fmt.Errorf("Error creating directory %s: %s\n", dir, err.Error())
}
} else {
if !statData.IsDir() {
- logger.Errorf("The path %s is not a directory\n", dir)
- return
+ return fmt.Errorf("The path %s is not a directory\n", dir)
}
}
return
}
-func addContainerLine(logger *logrus.Logger, node interface{}, cr Dict, container Dict) (csv string, cost float64) {
+func addContainerLine(logger *logrus.Logger, node interface{}, cr, container map[string]interface{}) (csv string, cost float64) {
csv = cr["uuid"].(string) + ","
csv += cr["name"].(string) + ","
csv += container["uuid"].(string) + ","
@@ -219,7 +215,7 @@ func addContainerLine(logger *logrus.Logger, node interface{}, cr Dict, containe
return
}
-func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload bool, object Dict) {
+func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload bool, object map[string]interface{}) {
reload = true
// See if we have a cached copy of this object
if _, err := os.Stat(file); err == nil {
@@ -248,7 +244,7 @@ func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload b
}
// Load an Arvados object.
-func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string) (object Dict, err error) {
+func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string, cache bool) (object map[string]interface{}, err error) {
err = ensureDirectory(logger, path)
if err != nil {
return
@@ -257,112 +253,118 @@ func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path st
file := path + "/" + uuid + ".json"
var reload bool
- reload, object = loadCachedObject(logger, file, uuid)
+ if !cache {
+ reload = true
+ } else {
+ reload, object = loadCachedObject(logger, file, uuid)
+ }
+ if !reload {
+ return
+ }
- if reload {
- var err error
- if strings.Contains(uuid, "-j7d0g-") {
- err = arv.Get("groups", uuid, nil, &object)
- } else if strings.Contains(uuid, "-xvhdp-") {
- err = arv.Get("container_requests", uuid, nil, &object)
- } else if strings.Contains(uuid, "-dz642-") {
- err = arv.Get("containers", uuid, nil, &object)
- } else {
- err = arv.Get("jobs", uuid, nil, &object)
- }
- if err != nil {
- logger.Fatalf("Error loading object with UUID %q:\n %s\n", uuid, err)
- }
- encoded, err := json.MarshalIndent(object, "", " ")
- if err != nil {
- logger.Fatalf("Error marshaling object with UUID %q:\n %s\n", uuid, err)
- }
- err = ioutil.WriteFile(file, encoded, 0644)
- if err != nil {
- logger.Fatalf("Error writing file %s:\n %s\n", file, err)
- }
+ if strings.Contains(uuid, "-j7d0g-") {
+ err = arv.Get("groups", uuid, nil, &object)
+ } else if strings.Contains(uuid, "-xvhdp-") {
+ err = arv.Get("container_requests", uuid, nil, &object)
+ } else if strings.Contains(uuid, "-dz642-") {
+ err = arv.Get("containers", uuid, nil, &object)
+ } else {
+ err = arv.Get("jobs", uuid, nil, &object)
+ }
+ if err != nil {
+ err = fmt.Errorf("Error loading object with UUID %q:\n %s\n", uuid, err)
+ return
+ }
+ encoded, err := json.MarshalIndent(object, "", " ")
+ if err != nil {
+ err = fmt.Errorf("Error marshaling object with UUID %q:\n %s\n", uuid, err)
+ return
+ }
+ err = ioutil.WriteFile(file, encoded, 0644)
+ if err != nil {
+ err = fmt.Errorf("Error writing file %s:\n %s\n", file, err)
+ return
}
return
}
-func getNode(arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, itemMap Dict) (node interface{}, err error) {
- if _, ok := itemMap["log_uuid"]; ok {
- if itemMap["log_uuid"] == nil {
- err = errors.New("No log collection")
- return
- }
+func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, itemMap map[string]interface{}) (node interface{}, err error) {
+ logUuid, ok := itemMap["log_uuid"]
+ if !ok {
+ err = errors.New("No log collection")
+ return
+ }
- var collection arvados.Collection
- err = arv.Get("collections", itemMap["log_uuid"].(string), nil, &collection)
- if err != nil {
- err = fmt.Errorf("Error getting collection: %s", err)
- return
- }
+ var collection arvados.Collection
+ err = arv.Get("collections", logUuid.(string), nil, &collection)
+ if err != nil {
+ err = fmt.Errorf("Error getting collection: %s", err)
+ return
+ }
- var fs arvados.CollectionFileSystem
- fs, err = collection.FileSystem(arv2, kc)
- if err != nil {
- err = fmt.Errorf("Error opening collection as filesystem: %s", err)
- return
- }
- var f http.File
- f, err = fs.Open("node.json")
+ var fs arvados.CollectionFileSystem
+ fs, err = collection.FileSystem(ac, kc)
+ if err != nil {
+ err = fmt.Errorf("Error opening collection as filesystem: %s", err)
+ return
+ }
+ var f http.File
+ f, err = fs.Open("node.json")
+ if err != nil {
+ err = fmt.Errorf("Error opening file 'node.json' in collection %s: %s", logUuid.(string), err)
+ return
+ }
+
+ var nodeDict map[string]interface{}
+ buf := new(bytes.Buffer)
+ _, err = buf.ReadFrom(f)
+ if err != nil {
+ err = fmt.Errorf("Error reading file 'node.json' in collection %s: %s", logUuid.(string), err)
+ return
+ }
+ contents := buf.String()
+ f.Close()
+
+ err = json.Unmarshal([]byte(contents), &nodeDict)
+ if err != nil {
+ err = fmt.Errorf("Error unmarshalling: %s", err)
+ return
+ }
+ if val, ok := nodeDict["properties"]; ok {
+ var encoded []byte
+ encoded, err = json.MarshalIndent(val, "", " ")
if err != nil {
- err = fmt.Errorf("Error opening file 'node.json' in collection %s: %s", itemMap["log_uuid"].(string), err)
+ err = fmt.Errorf("Error marshalling: %s", err)
return
}
-
- var nodeDict Dict
- buf := new(bytes.Buffer)
- _, err = buf.ReadFrom(f)
+ // node is type LegacyNodeInfo
+ var newNode LegacyNodeInfo
+ err = json.Unmarshal(encoded, &newNode)
if err != nil {
- err = fmt.Errorf("Error reading file 'node.json' in collection %s: %s", itemMap["log_uuid"].(string), err)
+ err = fmt.Errorf("Error unmarshalling: %s", err)
return
}
- contents := buf.String()
- f.Close()
-
- err = json.Unmarshal([]byte(contents), &nodeDict)
+ node = newNode
+ } else {
+ // node is type Node
+ var newNode Node
+ err = json.Unmarshal([]byte(contents), &newNode)
if err != nil {
err = fmt.Errorf("Error unmarshalling: %s", err)
return
}
- if val, ok := nodeDict["properties"]; ok {
- var encoded []byte
- encoded, err = json.MarshalIndent(val, "", " ")
- if err != nil {
- err = fmt.Errorf("Error marshalling: %s", err)
- return
- }
- // node is type LegacyNodeInfo
- var newNode LegacyNodeInfo
- err = json.Unmarshal(encoded, &newNode)
- if err != nil {
- err = fmt.Errorf("Error unmarshalling: %s", err)
- return
- }
- node = newNode
- } else {
- // node is type Node
- var newNode Node
- err = json.Unmarshal([]byte(contents), &newNode)
- if err != nil {
- err = fmt.Errorf("Error unmarshalling: %s", err)
- return
- }
- node = newNode
- }
+ node = newNode
}
return
}
-func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, resultsDir string) (cost map[string]float64) {
+func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
cost = make(map[string]float64)
- project, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+ project, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid, cache)
if err != nil {
- logger.Fatalf("Error loading object %s: %s\n", uuid, err.Error())
+ return nil, fmt.Errorf("Error loading object %s: %s\n", uuid, err.Error())
}
// arv -f uuid container_request list --filters '[["owner_uuid","=","<someuuid>"],["requesting_container_uuid","=",null]]'
@@ -381,16 +383,23 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
Operand: nil,
},
}
- err = arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs)
+ err = ac.RequestAndDecodeContext(context.Background(), &childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
+ "filters": filterset,
+ "limit": 10000,
+ })
if err != nil {
- logger.Fatalf("Error querying container_requests: %s\n", err.Error())
+ return nil, fmt.Errorf("Error querying container_requests: %s\n", err.Error())
}
if value, ok := childCrs["items"]; ok {
logger.Infof("Collecting top level container requests in project %s\n", uuid)
items := value.([]interface{})
for _, item := range items {
itemMap := item.(map[string]interface{})
- for k, v := range generateCrCsv(logger, itemMap["uuid"].(string), arv, arv2, kc, resultsDir) {
+ crCsv, err := generateCrCsv(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache)
+ if err != nil {
+ return nil, fmt.Errorf("Error generating container_request CSV: %s\n", err.Error())
+ }
+ for k, v := range crCsv {
cost[k] = v
}
}
@@ -400,7 +409,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
return
}
-func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, resultsDir string) (cost map[string]float64) {
+func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
cost = make(map[string]float64)
@@ -410,23 +419,22 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
var totalCost float64
// This is a container request, find the container
- cr, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+ cr, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid, cache)
if err != nil {
- log.Fatalf("Error loading object %s: %s", uuid, err)
+ return nil, fmt.Errorf("Error loading object %s: %s", uuid, err)
}
- container, err := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string))
+ container, err := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string), cache)
if err != nil {
- log.Fatalf("Error loading object %s: %s", cr["container_uuid"].(string), err)
+ return nil, fmt.Errorf("Error loading object %s: %s", cr["container_uuid"].(string), err)
}
- topNode, err := getNode(arv, arv2, kc, cr)
+ topNode, err := getNode(arv, ac, kc, cr)
if err != nil {
- log.Fatalf("Error getting node %s: %s\n", cr["uuid"], err)
+ return nil, fmt.Errorf("Error getting node %s: %s\n", cr["uuid"], err)
}
tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
csv += tmpCsv
totalCost += tmpTotalCost
-
cost[container["uuid"].(string)] = totalCost
// Now find all container requests that have the container we found above as requesting_container_uuid
@@ -437,9 +445,12 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
Operator: "=",
Operand: container["uuid"].(string),
}}
- err = arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs)
+ err = ac.RequestAndDecodeContext(context.Background(), &childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
+ "filters": filterset,
+ "limit": 10000,
+ })
if err != nil {
- log.Fatal("error querying container_requests", err.Error())
+ return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
}
if value, ok := childCrs["items"]; ok {
logger.Infof("Collecting child containers for container request %s", uuid)
@@ -447,14 +458,14 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
for _, item := range items {
logger.Info(".")
itemMap := item.(map[string]interface{})
- node, err := getNode(arv, arv2, kc, itemMap)
+ node, err := getNode(arv, ac, kc, itemMap)
if err != nil {
- log.Fatalf("Error getting node %s: %s\n", itemMap["uuid"], err)
+ return nil, fmt.Errorf("Error getting node %s: %s\n", itemMap["uuid"], err)
}
logger.Debug("\nChild container: " + itemMap["container_uuid"].(string) + "\n")
- c2, err := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string))
+ c2, err := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string), cache)
if err != nil {
- log.Fatalf("Error loading object %s: %s", cr["container_uuid"].(string), err)
+ return nil, fmt.Errorf("Error loading object %s: %s", cr["container_uuid"].(string), err)
}
tmpCsv, tmpTotalCost = addContainerLine(logger, node, itemMap, c2)
cost[itemMap["container_uuid"].(string)] = tmpTotalCost
@@ -470,20 +481,20 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
fName := resultsDir + "/" + uuid + ".csv"
err = ioutil.WriteFile(fName, []byte(csv), 0644)
if err != nil {
- logger.Errorf("Error writing file with path %s: %s\n", fName, err.Error())
- os.Exit(1)
+ return nil, fmt.Errorf("Error writing file with path %s: %s\n", fName, err.Error())
}
return
}
func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int) {
- exitcode, uuids, resultsDir := parseFlags(prog, args, loader, logger, stderr)
+ exitcode, uuids, resultsDir, cache := parseFlags(prog, args, loader, logger, stderr)
if exitcode != 0 {
return
}
err := ensureDirectory(logger, resultsDir)
if err != nil {
+ logger.Errorf("%s", err)
exitcode = 3
return
}
@@ -492,26 +503,39 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
logger.Errorf("error creating Arvados object: %s", err)
- os.Exit(1)
+ exitcode = 1
+ return
}
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
logger.Errorf("error creating Keep object: %s", err)
- os.Exit(1)
+ exitcode = 1
+ return
}
- arv2 := arvados.NewClientFromEnv()
+ ac := arvados.NewClientFromEnv()
cost := make(map[string]float64)
for _, uuid := range uuids {
if strings.Contains(uuid, "-j7d0g-") {
// This is a project (group)
- for k, v := range handleProject(logger, uuid, arv, arv2, kc, resultsDir) {
+ cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
+ if err != nil {
+ // FIXME print error
+ logger.Info(err.Error())
+ exitcode = 1
+ return
+ }
+ for k, v := range cost {
cost[k] = v
}
} else if strings.Contains(uuid, "-xvhdp-") {
// This is a container request
- for k, v := range generateCrCsv(logger, uuid, arv, arv2, kc, resultsDir) {
+ crCsv, err := generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
+ if err != nil {
+ logger.Fatalf("Error generating container_request CSV: %s\n", err.Error())
+ }
+ for k, v := range crCsv {
cost[k] = v
}
} else if strings.Contains(uuid, "-tpzed-") {
@@ -552,7 +576,8 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
err = ioutil.WriteFile(aFile, []byte(csv), 0644)
if err != nil {
logger.Errorf("Error writing file with path %s: %s\n", aFile, err.Error())
- os.Exit(1)
+ exitcode = 1
+ return
} else {
logger.Infof("\nAggregate cost accounting for all supplied uuids in %s\n", aFile)
}
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
index 2ef8733b0..0a44be8d8 100644
--- a/lib/costanalyzer/costanalyzer_test.go
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -6,6 +6,7 @@ package costanalyzer
import (
"bytes"
+ "context"
"io"
"io/ioutil"
"os"
@@ -105,13 +106,13 @@ func (s *Suite) SetUpSuite(c *check.C) {
func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) {
// Get the CR
var cr arvados.ContainerRequest
- err := arv.Get("container_requests", crUUID, arvadosclient.Dict{}, &cr)
+ err := ac.RequestAndDecodeContext(context.Background(), &cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil)
c.Assert(err, check.Equals, nil)
c.Assert(cr.LogUUID, check.Equals, logUUID)
// Get the log collection
var coll arvados.Collection
- err = arv.Get("collections", cr.LogUUID, arvadosclient.Dict{}, &coll)
+ err = ac.RequestAndDecodeContext(context.Background(), &coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
c.Assert(err, check.IsNil)
// Create a node.json file -- the fixture doesn't actually contain the contents of the collection.
@@ -130,7 +131,11 @@ func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Cl
c.Assert(mtxt, check.NotNil)
// Update collection record
- err = arv.Update("collections", cr.LogUUID, arvadosclient.Dict{"collection": arvadosclient.Dict{"manifest_text": mtxt}}, &coll)
+ err = ac.RequestAndDecodeContext(context.Background(), &coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "manifest_text": mtxt,
+ },
+ })
c.Assert(err, check.IsNil)
}
@@ -147,13 +152,13 @@ func (*Suite) TestContainerRequestUUID(c *check.C) {
// Run costanalyzer with 1 container request uuid
exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
@@ -166,7 +171,7 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
// Run costanalyzer with 2 container request uuids
exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
@@ -177,28 +182,36 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+ stdout.Truncate(0)
+ stderr.Truncate(0)
// Now move both container requests into an existing project, and then re-run
// the analysis with the project uuid. The results should be identical.
- arv, err := arvadosclient.MakeArvadosClient()
- c.Assert(err, check.Equals, nil)
-
+ ac := arvados.NewClientFromEnv()
var cr arvados.ContainerRequest
- err = arv.Update("container_requests", arvadostest.CompletedContainerRequestUUID, arvadosclient.Dict{"container_request": arvadosclient.Dict{"owner_uuid": arvadostest.AProjectUUID}}, &cr)
+ err = ac.RequestAndDecodeContext(context.Background(), &cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{
+ "container_request": map[string]interface{}{
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
c.Assert(err, check.IsNil)
- err = arv.Update("container_requests", arvadostest.CompletedContainerRequestUUID2, arvadosclient.Dict{"container_request": arvadosclient.Dict{"owner_uuid": arvadostest.AProjectUUID}}, &cr)
+ err = ac.RequestAndDecodeContext(context.Background(), &cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{
+ "container_request": map[string]interface{}{
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
c.Assert(err, check.IsNil)
// Run costanalyzer with the project uuid
- exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID, "-cache=false", "-log-level", "debug"}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
uuidReport, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
c.Assert(err, check.IsNil)
@@ -209,7 +222,7 @@ func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
re = regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches = re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches = re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err = ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
@@ -222,7 +235,7 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
// Run costanalyzer with 2 container request uuids
exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
- c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+ c.Assert(stdout.String(), check.Matches, "(?ms).*supplied uuids in .*")
uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
c.Assert(err, check.IsNil)
@@ -233,7 +246,7 @@ func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00586435")
re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
- matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+ matches := re.FindStringSubmatch(stdout.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
aggregateCostReport, err := ioutil.ReadFile(matches[1])
c.Assert(err, check.IsNil)
commit f6c012b2e08779385aaf98c15069febb8d873820
Author: Ward Vandewege <ward at curii.com>
Date: Tue Nov 3 16:34:16 2020 -0500
16950: add tests for the costanalyzer.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/costanalyzer/command.go b/lib/costanalyzer/command.go
index 3cca16ea0..0760b4fd2 100644
--- a/lib/costanalyzer/command.go
+++ b/lib/costanalyzer/command.go
@@ -22,7 +22,7 @@ func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
return []byte(entry.Message), nil
}
-// RunCommand implements the subcommand "deduplication-report <collection> <collection> ..."
+// RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
var err error
logger := ctxlog.New(stderr, "text", "info")
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
index d754e8875..c86e26769 100644
--- a/lib/costanalyzer/costanalyzer.go
+++ b/lib/costanalyzer/costanalyzer.go
@@ -119,7 +119,9 @@ Usage:
provider.
- when generating reports for older container requests, the cost data in the
Arvados API configuration file may have changed since the container request
- was fulfilled.
+ was fulfilled. This program uses the cost data stored at the time of the
+ execution of the container, stored in the 'node.json' file in its log
+ collection.
In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
ARVADOS_API_TOKEN environment variables must be set.
@@ -133,7 +135,7 @@ Options:
flags.Var(&uuids, "uuid", "Toplevel project or container request uuid. May be specified more than once.")
err := flags.Parse(args)
if err == flag.ErrHelp {
- exitCode = 0
+ exitCode = 1
return
} else if err != nil {
exitCode = 2
@@ -156,20 +158,21 @@ Options:
return
}
-func ensureDirectory(logger *logrus.Logger, dir string) {
+func ensureDirectory(logger *logrus.Logger, dir string) (err error) {
statData, err := os.Stat(dir)
if os.IsNotExist(err) {
err = os.MkdirAll(dir, 0700)
if err != nil {
logger.Errorf("Error creating directory %s: %s\n", dir, err.Error())
- os.Exit(1)
+ return
}
} else {
if !statData.IsDir() {
logger.Errorf("The path %s is not a directory\n", dir)
- os.Exit(1)
+ return
}
}
+ return
}
func addContainerLine(logger *logrus.Logger, node interface{}, cr Dict, container Dict) (csv string, cost float64) {
@@ -245,9 +248,11 @@ func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload b
}
// Load an Arvados object.
-func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string) (object Dict) {
-
- ensureDirectory(logger, path)
+func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string) (object Dict, err error) {
+ err = ensureDirectory(logger, path)
+ if err != nil {
+ return
+ }
file := path + "/" + uuid + ".json"
@@ -256,9 +261,7 @@ func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path st
if reload {
var err error
- if strings.Contains(uuid, "-d1hrv-") {
- err = arv.Get("pipeline_instances", uuid, nil, &object)
- } else if strings.Contains(uuid, "-j7d0g-") {
+ if strings.Contains(uuid, "-j7d0g-") {
err = arv.Get("groups", uuid, nil, &object)
} else if strings.Contains(uuid, "-xvhdp-") {
err = arv.Get("container_requests", uuid, nil, &object)
@@ -268,24 +271,21 @@ func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path st
err = arv.Get("jobs", uuid, nil, &object)
}
if err != nil {
- logger.Errorf("Error loading object with UUID %q:\n %s\n", uuid, err)
- os.Exit(1)
+ logger.Fatalf("Error loading object with UUID %q:\n %s\n", uuid, err)
}
encoded, err := json.MarshalIndent(object, "", " ")
if err != nil {
- logger.Errorf("Error marshaling object with UUID %q:\n %s\n", uuid, err)
- os.Exit(1)
+ logger.Fatalf("Error marshaling object with UUID %q:\n %s\n", uuid, err)
}
err = ioutil.WriteFile(file, encoded, 0644)
if err != nil {
- logger.Errorf("Error writing file %s:\n %s\n", file, err)
- os.Exit(1)
+ logger.Fatalf("Error writing file %s:\n %s\n", file, err)
}
}
return
}
-func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, itemMap Dict) (node interface{}, err error) {
+func getNode(arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, itemMap Dict) (node interface{}, err error) {
if _, ok := itemMap["log_uuid"]; ok {
if itemMap["log_uuid"] == nil {
err = errors.New("No log collection")
@@ -295,29 +295,28 @@ func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arva
var collection arvados.Collection
err = arv.Get("collections", itemMap["log_uuid"].(string), nil, &collection)
if err != nil {
- logger.Errorf("error getting collection: %s\n", err)
+ err = fmt.Errorf("Error getting collection: %s", err)
return
}
var fs arvados.CollectionFileSystem
fs, err = collection.FileSystem(arv2, kc)
if err != nil {
- logger.Errorf("error opening collection as filesystem: %s\n", err)
+ err = fmt.Errorf("Error opening collection as filesystem: %s", err)
return
}
var f http.File
f, err = fs.Open("node.json")
if err != nil {
- logger.Errorf("error opening file in collection: %s\n", err)
+ err = fmt.Errorf("Error opening file 'node.json' in collection %s: %s", itemMap["log_uuid"].(string), err)
return
}
var nodeDict Dict
- // TODO: checkout io (ioutil?) readall function
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(f)
if err != nil {
- logger.Errorf("error reading %q: %s\n", f, err)
+ err = fmt.Errorf("Error reading file 'node.json' in collection %s: %s", itemMap["log_uuid"].(string), err)
return
}
contents := buf.String()
@@ -325,21 +324,21 @@ func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arva
err = json.Unmarshal([]byte(contents), &nodeDict)
if err != nil {
- logger.Errorf("error unmarshalling: %s\n", err)
+ err = fmt.Errorf("Error unmarshalling: %s", err)
return
}
if val, ok := nodeDict["properties"]; ok {
var encoded []byte
encoded, err = json.MarshalIndent(val, "", " ")
if err != nil {
- logger.Errorf("error marshalling: %s\n", err)
+ err = fmt.Errorf("Error marshalling: %s", err)
return
}
// node is type LegacyNodeInfo
var newNode LegacyNodeInfo
err = json.Unmarshal(encoded, &newNode)
if err != nil {
- logger.Errorf("error unmarshalling: %s\n", err)
+ err = fmt.Errorf("Error unmarshalling: %s", err)
return
}
node = newNode
@@ -348,7 +347,7 @@ func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arva
var newNode Node
err = json.Unmarshal([]byte(contents), &newNode)
if err != nil {
- logger.Errorf("error unmarshalling: %s\n", err)
+ err = fmt.Errorf("Error unmarshalling: %s", err)
return
}
node = newNode
@@ -361,7 +360,10 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
cost = make(map[string]float64)
- project := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+ project, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+ if err != nil {
+ logger.Fatalf("Error loading object %s: %s\n", uuid, err.Error())
+ }
// arv -f uuid container_request list --filters '[["owner_uuid","=","<someuuid>"],["requesting_container_uuid","=",null]]'
@@ -379,7 +381,7 @@ func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
Operand: nil,
},
}
- err := arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs)
+ err = arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs)
if err != nil {
logger.Fatalf("Error querying container_requests: %s\n", err.Error())
}
@@ -408,12 +410,18 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
var totalCost float64
// This is a container request, find the container
- cr := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
- container := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string))
+ cr, err := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+ if err != nil {
+ log.Fatalf("Error loading object %s: %s", uuid, err)
+ }
+ container, err := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string))
+ if err != nil {
+ log.Fatalf("Error loading object %s: %s", cr["container_uuid"].(string), err)
+ }
- topNode, err := getNode(logger, arv, arv2, kc, cr)
+ topNode, err := getNode(arv, arv2, kc, cr)
if err != nil {
- log.Fatalf("error getting node: %s", err)
+ log.Fatalf("Error getting node %s: %s\n", cr["uuid"], err)
}
tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
csv += tmpCsv
@@ -439,9 +447,15 @@ func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.Arvado
for _, item := range items {
logger.Info(".")
itemMap := item.(map[string]interface{})
- node, _ := getNode(logger, arv, arv2, kc, itemMap)
+ node, err := getNode(arv, arv2, kc, itemMap)
+ if err != nil {
+ log.Fatalf("Error getting node %s: %s\n", itemMap["uuid"], err)
+ }
logger.Debug("\nChild container: " + itemMap["container_uuid"].(string) + "\n")
- c2 := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string))
+ c2, err := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string))
+ if err != nil {
+ log.Fatalf("Error loading object %s: %s", cr["container_uuid"].(string), err)
+ }
tmpCsv, tmpTotalCost = addContainerLine(logger, node, itemMap, c2)
cost[itemMap["container_uuid"].(string)] = tmpTotalCost
csv += tmpCsv
@@ -468,8 +482,11 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
if exitcode != 0 {
return
}
-
- ensureDirectory(logger, resultsDir)
+ err := ensureDirectory(logger, resultsDir)
+ if err != nil {
+ exitcode = 3
+ return
+ }
// Arvados Client setup
arv, err := arvadosclient.MakeArvadosClient()
@@ -486,23 +503,7 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
arv2 := arvados.NewClientFromEnv()
cost := make(map[string]float64)
-
for _, uuid := range uuids {
- //csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
-
- if strings.Contains(uuid, "-d1hrv-") {
- // This is a pipeline instance, not a job! Find the cwl-runner job.
- pi := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
- for _, v := range pi["components"].(map[string]interface{}) {
- x := v.(map[string]interface{})
- y := x["job"].(map[string]interface{})
- uuid = y["uuid"].(string)
- }
- }
-
- // for projects:
- // arv -f uuid container_request list --filters '[["owner_uuid","=","<someuuid>"],["requesting_container_uuid","=",null]]'
-
if strings.Contains(uuid, "-j7d0g-") {
// This is a project (group)
for k, v := range handleProject(logger, uuid, arv, arv2, kc, resultsDir) {
@@ -528,7 +529,7 @@ func costanalyzer(prog string, args []string, loader *config.Loader, logger *log
if len(cost) == 0 {
logger.Info("Nothing to do!\n")
- os.Exit(0)
+ return
}
var csv string
diff --git a/lib/costanalyzer/costanalyzer_test.go b/lib/costanalyzer/costanalyzer_test.go
new file mode 100644
index 000000000..2ef8733b0
--- /dev/null
+++ b/lib/costanalyzer/costanalyzer_test.go
@@ -0,0 +1,242 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package costanalyzer
+
+import (
+ "bytes"
+ "io"
+ "io/ioutil"
+ "os"
+ "regexp"
+ "testing"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (s *Suite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the database so they don't affect subsequent tests.
+ arvadostest.ResetEnv()
+}
+
+func (s *Suite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+ arvadostest.StartKeep(2, true)
+
+ // Get the various arvados, arvadosclient, and keep client objects
+ ac := arvados.NewClientFromEnv()
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = arvadostest.ActiveToken
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.Equals, nil)
+
+ standardE4sV3JSON := `{
+ "Name": "Standard_E4s_v3",
+ "ProviderType": "Standard_E4s_v3",
+ "VCPUs": 4,
+ "RAM": 34359738368,
+ "Scratch": 64000000000,
+ "IncludedScratch": 64000000000,
+ "AddedScratch": 0,
+ "Price": 0.292,
+ "Preemptible": false
+}`
+ standardD32sV3JSON := `{
+ "Name": "Standard_D32s_v3",
+ "ProviderType": "Standard_D32s_v3",
+ "VCPUs": 32,
+ "RAM": 137438953472,
+ "Scratch": 256000000000,
+ "IncludedScratch": 256000000000,
+ "AddedScratch": 0,
+ "Price": 1.76,
+ "Preemptible": false
+}`
+
+ standardA1V2JSON := `{
+ "Name": "a1v2",
+ "ProviderType": "Standard_A1_v2",
+ "VCPUs": 1,
+ "RAM": 2147483648,
+ "Scratch": 10000000000,
+ "IncludedScratch": 10000000000,
+ "AddedScratch": 0,
+ "Price": 0.043,
+ "Preemptible": false
+}`
+
+ standardA2V2JSON := `{
+ "Name": "a2v2",
+ "ProviderType": "Standard_A2_v2",
+ "VCPUs": 2,
+ "RAM": 4294967296,
+ "Scratch": 20000000000,
+ "IncludedScratch": 20000000000,
+ "AddedScratch": 0,
+ "Price": 0.091,
+ "Preemptible": false
+}`
+
+ // Our fixtures do not actually contain file contents. Populate the log collections we're going to use with the node.json file
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID, arvadostest.LogCollectionUUID, standardE4sV3JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID2, arvadostest.LogCollectionUUID2, standardD32sV3JSON)
+
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.DiagnosticsContainerRequest1LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest2UUID, arvadostest.DiagnosticsContainerRequest2LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher1ContainerRequestUUID, arvadostest.Hasher1LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher2ContainerRequestUUID, arvadostest.Hasher2LogCollectionUUID, standardA2V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, standardA1V2JSON)
+}
+
+func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) {
+ // Get the CR
+ var cr arvados.ContainerRequest
+ err := arv.Get("container_requests", crUUID, arvadosclient.Dict{}, &cr)
+ c.Assert(err, check.Equals, nil)
+ c.Assert(cr.LogUUID, check.Equals, logUUID)
+
+ // Get the log collection
+ var coll arvados.Collection
+ err = arv.Get("collections", cr.LogUUID, arvadosclient.Dict{}, &coll)
+ c.Assert(err, check.IsNil)
+
+ // Create a node.json file -- the fixture doesn't actually contain the contents of the collection.
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("node.json", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, nodeJSON)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+
+ // Flush the data to Keep
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ c.Assert(mtxt, check.NotNil)
+
+ // Update collection record
+ err = arv.Update("collections", cr.LogUUID, arvadosclient.Dict{"collection": arvadosclient.Dict{"manifest_text": mtxt}}, &coll)
+ c.Assert(err, check.IsNil)
+}
+
+func (*Suite) TestUsage(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-help", "-log-level=debug"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 1)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`)
+}
+
+func (*Suite) TestContainerRequestUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 1 container request uuid
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+}
+
+func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedContainerRequestUUID, "-uuid", arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+
+ uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+
+ // Now move both container requests into an existing project, and then re-run
+ // the analysis with the project uuid. The results should be identical.
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+
+ var cr arvados.ContainerRequest
+ err = arv.Update("container_requests", arvadostest.CompletedContainerRequestUUID, arvadosclient.Dict{"container_request": arvadosclient.Dict{"owner_uuid": arvadostest.AProjectUUID}}, &cr)
+ c.Assert(err, check.IsNil)
+ err = arv.Update("container_requests", arvadostest.CompletedContainerRequestUUID2, arvadosclient.Dict{"container_request": arvadosclient.Dict{"owner_uuid": arvadostest.AProjectUUID}}, &cr)
+ c.Assert(err, check.IsNil)
+
+ // Run costanalyzer with the project uuid
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+
+ uuidReport2, err = ioutil.ReadFile("results/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+
+ re = regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches = re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err = ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+}
+
+func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-uuid", arvadostest.CompletedDiagnosticsContainerRequest1UUID, "-uuid", arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00914539")
+
+ uuidReport2, err := ioutil.ReadFile("results/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00586435")
+
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01490377")
+}
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 9049c73c4..aeb5a47e6 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -44,8 +44,27 @@ const (
RunningContainerUUID = "zzzzz-dz642-runningcontainr"
- CompletedContainerUUID = "zzzzz-dz642-compltcontainer"
- CompletedContainerRequestUUID = "zzzzz-xvhdp-cr4completedctr"
+ CompletedContainerUUID = "zzzzz-dz642-compltcontainer"
+ CompletedContainerRequestUUID = "zzzzz-xvhdp-cr4completedctr"
+ CompletedContainerRequestUUID2 = "zzzzz-xvhdp-cr4completedcr2"
+
+ CompletedDiagnosticsContainerRequest1UUID = "zzzzz-xvhdp-diagnostics0001"
+ CompletedDiagnosticsContainerRequest2UUID = "zzzzz-xvhdp-diagnostics0002"
+ CompletedDiagnosticsContainer1UUID = "zzzzz-dz642-diagcompreq0001"
+ CompletedDiagnosticsContainer2UUID = "zzzzz-dz642-diagcompreq0002"
+ DiagnosticsContainerRequest1LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog1"
+ DiagnosticsContainerRequest2LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog2"
+
+ CompletedDiagnosticsHasher1ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0001"
+ CompletedDiagnosticsHasher2ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0002"
+ CompletedDiagnosticsHasher3ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0003"
+ CompletedDiagnosticsHasher1ContainerUUID = "zzzzz-dz642-diagcomphasher1"
+ CompletedDiagnosticsHasher2ContainerUUID = "zzzzz-dz642-diagcomphasher2"
+ CompletedDiagnosticsHasher3ContainerUUID = "zzzzz-dz642-diagcomphasher3"
+
+ Hasher1LogCollectionUUID = "zzzzz-4zz18-dlogcollhash001"
+ Hasher2LogCollectionUUID = "zzzzz-4zz18-dlogcollhash002"
+ Hasher3LogCollectionUUID = "zzzzz-4zz18-dlogcollhash003"
ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
ArvadosRepoName = "arvados"
@@ -75,7 +94,8 @@ const (
CollectionWithUniqueWordsUUID = "zzzzz-4zz18-mnt690klmb51aud"
- LogCollectionUUID = "zzzzz-4zz18-logcollection01"
+ LogCollectionUUID = "zzzzz-4zz18-logcollection01"
+ LogCollectionUUID2 = "zzzzz-4zz18-logcollection02"
)
// PathologicalManifest : A valid manifest designed to test
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 2243d6a44..767f035b8 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -1043,6 +1043,78 @@ log_collection:
manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
name: a real log collection for a completed container
+log_collection2:
+ uuid: zzzzz-4zz18-logcollection02
+ current_version_uuid: zzzzz-4zz18-logcollection02
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-10-29T00:51:44.075594000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-10-29T00:51:44.072109000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: another real log collection for a completed container
+
+diagnostics_request_container_log_collection:
+ uuid: zzzzz-4zz18-diagcompreqlog1
+ current_version_uuid: zzzzz-4zz18-diagcompreqlog1
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:44.007557000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:44.005381000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: Container log for request zzzzz-xvhdp-diagnostics0001
+
+hasher1_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash001
+ current_version_uuid: zzzzz-4zz18-dlogcollhash001
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:16:55.272606000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:16:55.267006000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher1 log collection
+
+hasher2_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash002
+ current_version_uuid: zzzzz-4zz18-dlogcollhash002
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:23.547251000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:23.545275000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher2 log collection
+
+hasher3_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash003
+ current_version_uuid: zzzzz-4zz18-dlogcollhash003
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:38.789204000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:38.787329000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher3 log collection
+
+diagnostics_request_container_log_collection2:
+ uuid: zzzzz-4zz18-diagcompreqlog2
+ current_version_uuid: zzzzz-4zz18-diagcompreqlog2
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-03T16:17:53.351593000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-03T16:17:53.346969000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: Container log for request zzzzz-xvhdp-diagnostics0002
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
diff --git a/services/api/test/fixtures/container_requests.yml b/services/api/test/fixtures/container_requests.yml
index 5816aa730..1c6260502 100644
--- a/services/api/test/fixtures/container_requests.yml
+++ b/services/api/test/fixtures/container_requests.yml
@@ -120,11 +120,239 @@ completed-older:
output_path: test
command: ["arvados-cwl-runner", "echo", "hello"]
container_uuid: zzzzz-dz642-compltcontainr2
+ log_uuid: zzzzz-4zz18-logcollection02
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
runtime_constraints:
vcpus: 1
ram: 123
mounts: {}
+completed_diagnostics:
+ name: CWL diagnostics hasher
+ uuid: zzzzz-xvhdp-diagnostics0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 1
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-diagcompreqlog1
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 1342177280
+ API: true
+
+completed_diagnostics_hasher1:
+ name: hasher1
+ uuid: zzzzz-xvhdp-diag1hasher0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher1
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher1
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash001
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher2:
+ name: hasher2
+ uuid: zzzzz-xvhdp-diag1hasher0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:17:07.067464000Z
+ modified_at: 2020-11-02T00:20:23.557498000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher2
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher2
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash002
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 2
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher3:
+ name: hasher3
+ uuid: zzzzz-xvhdp-diag1hasher0003
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:20:30.960251000Z
+ modified_at: 2020-11-02T00:20:38.799377000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher3
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher3
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash003
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics2:
+ name: Copy of CWL diagnostics hasher
+ uuid: zzzzz-xvhdp-diagnostics0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 1
+ created_at: 2020-11-03T15:54:30.098485000Z
+ modified_at: 2020-11-03T16:17:53.406809000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-diagcompreqlog2
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 1342177280
+ API: true
+
+completed_diagnostics_hasher1_reuse:
+ name: hasher1
+ uuid: zzzzz-xvhdp-diag2hasher0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher1
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher1
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash001
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher2_reuse:
+ name: hasher2
+ uuid: zzzzz-xvhdp-diag2hasher0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:17:07.067464000Z
+ modified_at: 2020-11-02T00:20:23.557498000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher2
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher2
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash002
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 2
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher3_reuse:
+ name: hasher3
+ uuid: zzzzz-xvhdp-diag2hasher0003
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:20:30.960251000Z
+ modified_at: 2020-11-02T00:20:38.799377000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher3
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher3
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash003
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
requester:
uuid: zzzzz-xvhdp-9zacv3o1xw6sxz5
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
diff --git a/services/api/test/fixtures/containers.yml b/services/api/test/fixtures/containers.yml
index f18adb5db..b7d082771 100644
--- a/services/api/test/fixtures/containers.yml
+++ b/services/api/test/fixtures/containers.yml
@@ -126,6 +126,153 @@ completed_older:
secret_mounts: {}
secret_mounts_md5: 99914b932bd37a50b983c5e7c90ae93b
+diagnostics_completed_requester:
+ uuid: zzzzz-dz642-diagcompreq0001
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:03:50.192697000Z
+ modified_at: 2020-11-02T00:20:43.987275000Z
+ started_at: 2020-11-02T00:08:07.186711000Z
+ finished_at: 2020-11-02T00:20:43.975416000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 6129e376cb05c942f75a0c36083383e8+244
+ output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 1342177280
+ vcpus: 1
+
+diagnostics_completed_hasher1:
+ uuid: zzzzz-dz642-diagcomphasher1
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:08:18.829222000Z
+ modified_at: 2020-11-02T00:16:55.142023000Z
+ started_at: 2020-11-02T00:16:52.375871000Z
+ finished_at: 2020-11-02T00:16:55.105985000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: fed8fb19fe8e3a320c29fed0edab12dd+220
+ output: d3a687732e84061f3bae15dc7e313483+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 1
+
+diagnostics_completed_hasher2:
+ uuid: zzzzz-dz642-diagcomphasher2
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:17:07.026493000Z
+ modified_at: 2020-11-02T00:20:23.505908000Z
+ started_at: 2020-11-02T00:20:21.513185000Z
+ finished_at: 2020-11-02T00:20:23.478317000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 4fc03b95fc2646b0dec7383dbb7d56d8+221
+ output: 6bd770f6cf8f83e7647c602eecfaeeb8+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 2
+
+diagnostics_completed_hasher3:
+ uuid: zzzzz-dz642-diagcomphasher3
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:20:30.943856000Z
+ modified_at: 2020-11-02T00:20:38.746541000Z
+ started_at: 2020-11-02T00:20:36.748957000Z
+ finished_at: 2020-11-02T00:20:38.732199000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 1eeaf70de0f65b1346e54c59f09e848d+210
+ output: 11b5fdaa380102e760c3eb6de80a9876+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 1
+
+diagnostics_completed_requester2:
+ uuid: zzzzz-dz642-diagcompreq0002
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 1124295487972526
+ created_at: 2020-11-03T15:54:36.504661000Z
+ modified_at: 2020-11-03T16:17:53.242868000Z
+ started_at: 2020-11-03T16:09:51.123659000Z
+ finished_at: 2020-11-03T16:17:53.220358000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: f1933bf5191f576613ea7f65bd0ead53+244
+ output: 941b71a57208741ce8742eca62352fb1+123
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 1342177280
+ vcpus: 1
+
requester:
uuid: zzzzz-dz642-requestingcntnr
owner_uuid: zzzzz-tpzed-000000000000000
commit 4bd8ba40bab2ddd229f921d2fa709e0c19edf371
Author: Ward Vandewege <ward at curii.com>
Date: Sat Oct 3 13:58:50 2020 -0400
16950: add the costanalyzer to arvados-client.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
index bcc3dda09..47fcd5ad7 100644
--- a/cmd/arvados-client/cmd.go
+++ b/cmd/arvados-client/cmd.go
@@ -9,6 +9,7 @@ import (
"git.arvados.org/arvados.git/lib/cli"
"git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/costanalyzer"
"git.arvados.org/arvados.git/lib/deduplicationreport"
"git.arvados.org/arvados.git/lib/mount"
)
@@ -55,6 +56,7 @@ var (
"mount": mount.Command,
"deduplication-report": deduplicationreport.Command,
+ "costanalyzer": costanalyzer.Command,
})
)
diff --git a/lib/costanalyzer/command.go b/lib/costanalyzer/command.go
new file mode 100644
index 000000000..3cca16ea0
--- /dev/null
+++ b/lib/costanalyzer/command.go
@@ -0,0 +1,43 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package costanalyzer
+
+import (
+ "io"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+var Command command
+
+type command struct{}
+
+type NoPrefixFormatter struct{}
+
+func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+ return []byte(entry.Message), nil
+}
+
+// RunCommand implements the subcommand "deduplication-report <collection> <collection> ..."
+func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ logger := ctxlog.New(stderr, "text", "info")
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Error("fatal")
+ }
+ }()
+
+ logger.SetFormatter(new(NoPrefixFormatter))
+
+ loader := config.NewLoader(stdin, logger)
+ loader.SkipLegacy = true
+
+ exitcode := costanalyzer(prog, args, loader, logger, stdout, stderr)
+
+ return exitcode
+}
diff --git a/lib/costanalyzer/costanalyzer.go b/lib/costanalyzer/costanalyzer.go
new file mode 100644
index 000000000..d754e8875
--- /dev/null
+++ b/lib/costanalyzer/costanalyzer.go
@@ -0,0 +1,559 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package costanalyzer
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ "io"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/sirupsen/logrus"
+)
+
+// Dict is a helper type so we don't have to write out 'map[string]interface{}' every time.
+type Dict map[string]interface{}
+
+// LegacyNodeInfo is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3)
+// Example:
+// {
+// "total_cpu_cores":2,
+// "total_scratch_mb":33770,
+// "cloud_node":
+// {
+// "price":0.1,
+// "size":"m4.large"
+// },
+// "total_ram_mb":7986
+// }
+type LegacyNodeInfo struct {
+ CPUCores int64 `json:"total_cpu_cores"`
+ ScratchMb int64 `json:"total_scratch_mb"`
+ RAMMb int64 `json:"total_ram_mb"`
+ CloudNode LegacyCloudNode `json:"cloud_node"`
+}
+
+// LegacyCloudNode is a struct for records created by Arvados Node Manager (Arvados <= 1.4.3)
+type LegacyCloudNode struct {
+ Price float64 `json:"price"`
+ Size string `json:"size"`
+}
+
+// Node is a struct for records created by Arvados Dispatch Cloud (Arvados >= 2.0.0)
+// Example:
+// {
+// "Name": "Standard_D1_v2",
+// "ProviderType": "Standard_D1_v2",
+// "VCPUs": 1,
+// "RAM": 3584000000,
+// "Scratch": 50000000000,
+// "IncludedScratch": 50000000000,
+// "AddedScratch": 0,
+// "Price": 0.057,
+// "Preemptible": false
+//}
+type Node struct {
+ VCPUs int64
+ Scratch int64
+ RAM int64
+ Price float64
+ Name string
+ ProviderType string
+ Preemptible bool
+}
+
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+ return ""
+}
+
+func (i *arrayFlags) Set(value string) error {
+ *i = append(*i, value)
+ return nil
+}
+
+func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string) {
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ flags.Usage = func() {
+ fmt.Fprintf(flags.Output(), `
+Usage:
+ %s [options ...]
+
+ This program analyzes the cost of Arvados container requests. For each uuid
+ supplied, it creates a CSV report that lists all the containers used to
+ fulfill the container request, together with the machine type and cost of
+ each container.
+
+ When supplied with the uuid of a container request, it will calculate the
+ cost of that container request and all its children. When suplied with a
+ project uuid or when supplied with multiple container request uuids, it will
+ create a CSV report for each supplied uuid, as well as a CSV file with
+ aggregate cost accounting for all supplied uuids. The aggregate cost report
+ takes container reuse into account: if a container was reused between several
+ container requests, its cost will only be counted once.
+
+ To get the node costs, the progam queries the Arvados API for current cost
+ data for each node type used. This means that the reported cost always
+ reflects the cost data as currently defined in the Arvados API configuration
+ file.
+
+ Caveats:
+ - the Arvados API configuration cost data may be out of sync with the cloud
+ provider.
+ - when generating reports for older container requests, the cost data in the
+ Arvados API configuration file may have changed since the container request
+ was fulfilled.
+
+ In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+ ARVADOS_API_TOKEN environment variables must be set.
+
+Options:
+`, prog)
+ flags.PrintDefaults()
+ }
+ loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)")
+ resultsDir = *flags.String("output", "results", "output directory for the CSV reports")
+ flags.Var(&uuids, "uuid", "Toplevel project or container request uuid. May be specified more than once.")
+ err := flags.Parse(args)
+ if err == flag.ErrHelp {
+ exitCode = 0
+ return
+ } else if err != nil {
+ exitCode = 2
+ return
+ }
+
+ if len(uuids) < 1 {
+ logger.Errorf("Error: no uuid(s) provided")
+ flags.Usage()
+ exitCode = 2
+ return
+ }
+
+ lvl, err := logrus.ParseLevel(*loglevel)
+ if err != nil {
+ exitCode = 2
+ return
+ }
+ logger.SetLevel(lvl)
+ return
+}
+
+func ensureDirectory(logger *logrus.Logger, dir string) {
+ statData, err := os.Stat(dir)
+ if os.IsNotExist(err) {
+ err = os.MkdirAll(dir, 0700)
+ if err != nil {
+ logger.Errorf("Error creating directory %s: %s\n", dir, err.Error())
+ os.Exit(1)
+ }
+ } else {
+ if !statData.IsDir() {
+ logger.Errorf("The path %s is not a directory\n", dir)
+ os.Exit(1)
+ }
+ }
+}
+
+func addContainerLine(logger *logrus.Logger, node interface{}, cr Dict, container Dict) (csv string, cost float64) {
+ csv = cr["uuid"].(string) + ","
+ csv += cr["name"].(string) + ","
+ csv += container["uuid"].(string) + ","
+ csv += container["state"].(string) + ","
+ if container["started_at"] != nil {
+ csv += container["started_at"].(string) + ","
+ } else {
+ csv += ","
+ }
+
+ var delta time.Duration
+ if container["finished_at"] != nil {
+ csv += container["finished_at"].(string) + ","
+ finishedTimestamp, err := time.Parse("2006-01-02T15:04:05.000000000Z", container["finished_at"].(string))
+ if err != nil {
+ fmt.Println(err)
+ }
+ startedTimestamp, err := time.Parse("2006-01-02T15:04:05.000000000Z", container["started_at"].(string))
+ if err != nil {
+ fmt.Println(err)
+ }
+ delta = finishedTimestamp.Sub(startedTimestamp)
+ csv += strconv.FormatFloat(delta.Seconds(), 'f', 0, 64) + ","
+ } else {
+ csv += ",,"
+ }
+ var price float64
+ var size string
+ switch n := node.(type) {
+ case Node:
+ price = n.Price
+ size = n.ProviderType
+ case LegacyNodeInfo:
+ price = n.CloudNode.Price
+ size = n.CloudNode.Size
+ default:
+ logger.Warn("WARNING: unknown node type found!")
+ }
+ cost = delta.Seconds() / 3600 * price
+ csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
+ return
+}
+
+func loadCachedObject(logger *logrus.Logger, file string, uuid string) (reload bool, object Dict) {
+ reload = true
+ // See if we have a cached copy of this object
+ if _, err := os.Stat(file); err == nil {
+ data, err := ioutil.ReadFile(file)
+ if err != nil {
+ logger.Errorf("error reading %q: %s", file, err)
+ return
+ }
+ err = json.Unmarshal(data, &object)
+ if err != nil {
+ logger.Errorf("failed to unmarshal json: %s: %s", data, err)
+ return
+ }
+
+ // See if it is in a final state, if that makes sense
+ // Projects (j7d0g) do not have state so they should always be reloaded
+ if !strings.Contains(uuid, "-j7d0g-") {
+ if object["state"].(string) == "Complete" || object["state"].(string) == "Failed" {
+ reload = false
+ logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
+ return
+ }
+ }
+ }
+ return
+}
+
+// Load an Arvados object.
+func loadObject(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, path string, uuid string) (object Dict) {
+
+ ensureDirectory(logger, path)
+
+ file := path + "/" + uuid + ".json"
+
+ var reload bool
+ reload, object = loadCachedObject(logger, file, uuid)
+
+ if reload {
+ var err error
+ if strings.Contains(uuid, "-d1hrv-") {
+ err = arv.Get("pipeline_instances", uuid, nil, &object)
+ } else if strings.Contains(uuid, "-j7d0g-") {
+ err = arv.Get("groups", uuid, nil, &object)
+ } else if strings.Contains(uuid, "-xvhdp-") {
+ err = arv.Get("container_requests", uuid, nil, &object)
+ } else if strings.Contains(uuid, "-dz642-") {
+ err = arv.Get("containers", uuid, nil, &object)
+ } else {
+ err = arv.Get("jobs", uuid, nil, &object)
+ }
+ if err != nil {
+ logger.Errorf("Error loading object with UUID %q:\n %s\n", uuid, err)
+ os.Exit(1)
+ }
+ encoded, err := json.MarshalIndent(object, "", " ")
+ if err != nil {
+ logger.Errorf("Error marshaling object with UUID %q:\n %s\n", uuid, err)
+ os.Exit(1)
+ }
+ err = ioutil.WriteFile(file, encoded, 0644)
+ if err != nil {
+ logger.Errorf("Error writing file %s:\n %s\n", file, err)
+ os.Exit(1)
+ }
+ }
+ return
+}
+
+func getNode(logger *logrus.Logger, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, itemMap Dict) (node interface{}, err error) {
+ if _, ok := itemMap["log_uuid"]; ok {
+ if itemMap["log_uuid"] == nil {
+ err = errors.New("No log collection")
+ return
+ }
+
+ var collection arvados.Collection
+ err = arv.Get("collections", itemMap["log_uuid"].(string), nil, &collection)
+ if err != nil {
+ logger.Errorf("error getting collection: %s\n", err)
+ return
+ }
+
+ var fs arvados.CollectionFileSystem
+ fs, err = collection.FileSystem(arv2, kc)
+ if err != nil {
+ logger.Errorf("error opening collection as filesystem: %s\n", err)
+ return
+ }
+ var f http.File
+ f, err = fs.Open("node.json")
+ if err != nil {
+ logger.Errorf("error opening file in collection: %s\n", err)
+ return
+ }
+
+ var nodeDict Dict
+ // TODO: checkout io (ioutil?) readall function
+ buf := new(bytes.Buffer)
+ _, err = buf.ReadFrom(f)
+ if err != nil {
+ logger.Errorf("error reading %q: %s\n", f, err)
+ return
+ }
+ contents := buf.String()
+ f.Close()
+
+ err = json.Unmarshal([]byte(contents), &nodeDict)
+ if err != nil {
+ logger.Errorf("error unmarshalling: %s\n", err)
+ return
+ }
+ if val, ok := nodeDict["properties"]; ok {
+ var encoded []byte
+ encoded, err = json.MarshalIndent(val, "", " ")
+ if err != nil {
+ logger.Errorf("error marshalling: %s\n", err)
+ return
+ }
+ // node is type LegacyNodeInfo
+ var newNode LegacyNodeInfo
+ err = json.Unmarshal(encoded, &newNode)
+ if err != nil {
+ logger.Errorf("error unmarshalling: %s\n", err)
+ return
+ }
+ node = newNode
+ } else {
+ // node is type Node
+ var newNode Node
+ err = json.Unmarshal([]byte(contents), &newNode)
+ if err != nil {
+ logger.Errorf("error unmarshalling: %s\n", err)
+ return
+ }
+ node = newNode
+ }
+ }
+ return
+}
+
+func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, resultsDir string) (cost map[string]float64) {
+
+ cost = make(map[string]float64)
+
+ project := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+
+ // arv -f uuid container_request list --filters '[["owner_uuid","=","<someuuid>"],["requesting_container_uuid","=",null]]'
+
+ // Now find all container requests that have the container we found above as requesting_container_uuid
+ var childCrs map[string]interface{}
+ filterset := []arvados.Filter{
+ {
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: project["uuid"].(string),
+ },
+ {
+ Attr: "requesting_container_uuid",
+ Operator: "=",
+ Operand: nil,
+ },
+ }
+ err := arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs)
+ if err != nil {
+ logger.Fatalf("Error querying container_requests: %s\n", err.Error())
+ }
+ if value, ok := childCrs["items"]; ok {
+ logger.Infof("Collecting top level container requests in project %s\n", uuid)
+ items := value.([]interface{})
+ for _, item := range items {
+ itemMap := item.(map[string]interface{})
+ for k, v := range generateCrCsv(logger, itemMap["uuid"].(string), arv, arv2, kc, resultsDir) {
+ cost[k] = v
+ }
+ }
+ } else {
+ logger.Infof("No top level container requests found in project %s\n", uuid)
+ }
+ return
+}
+
+func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, arv2 *arvados.Client, kc *keepclient.KeepClient, resultsDir string) (cost map[string]float64) {
+
+ cost = make(map[string]float64)
+
+ csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+ var tmpCsv string
+ var tmpTotalCost float64
+ var totalCost float64
+
+ // This is a container request, find the container
+ cr := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+ container := loadObject(logger, arv, resultsDir+"/"+uuid, cr["container_uuid"].(string))
+
+ topNode, err := getNode(logger, arv, arv2, kc, cr)
+ if err != nil {
+ log.Fatalf("error getting node: %s", err)
+ }
+ tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
+ csv += tmpCsv
+ totalCost += tmpTotalCost
+
+ cost[container["uuid"].(string)] = totalCost
+
+ // Now find all container requests that have the container we found above as requesting_container_uuid
+ var childCrs map[string]interface{}
+ filterset := []arvados.Filter{
+ {
+ Attr: "requesting_container_uuid",
+ Operator: "=",
+ Operand: container["uuid"].(string),
+ }}
+ err = arv.List("container_requests", arvadosclient.Dict{"filters": filterset, "limit": 10000}, &childCrs)
+ if err != nil {
+ log.Fatal("error querying container_requests", err.Error())
+ }
+ if value, ok := childCrs["items"]; ok {
+ logger.Infof("Collecting child containers for container request %s", uuid)
+ items := value.([]interface{})
+ for _, item := range items {
+ logger.Info(".")
+ itemMap := item.(map[string]interface{})
+ node, _ := getNode(logger, arv, arv2, kc, itemMap)
+ logger.Debug("\nChild container: " + itemMap["container_uuid"].(string) + "\n")
+ c2 := loadObject(logger, arv, resultsDir+"/"+uuid, itemMap["container_uuid"].(string))
+ tmpCsv, tmpTotalCost = addContainerLine(logger, node, itemMap, c2)
+ cost[itemMap["container_uuid"].(string)] = tmpTotalCost
+ csv += tmpCsv
+ totalCost += tmpTotalCost
+ }
+ }
+ logger.Info(" done\n")
+
+ csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
+
+ // Write the resulting CSV file
+ fName := resultsDir + "/" + uuid + ".csv"
+ err = ioutil.WriteFile(fName, []byte(csv), 0644)
+ if err != nil {
+ logger.Errorf("Error writing file with path %s: %s\n", fName, err.Error())
+ os.Exit(1)
+ }
+
+ return
+}
+
+func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int) {
+ exitcode, uuids, resultsDir := parseFlags(prog, args, loader, logger, stderr)
+ if exitcode != 0 {
+ return
+ }
+
+ ensureDirectory(logger, resultsDir)
+
+ // Arvados Client setup
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ logger.Errorf("error creating Arvados object: %s", err)
+ os.Exit(1)
+ }
+ kc, err := keepclient.MakeKeepClient(arv)
+ if err != nil {
+ logger.Errorf("error creating Keep object: %s", err)
+ os.Exit(1)
+ }
+
+ arv2 := arvados.NewClientFromEnv()
+
+ cost := make(map[string]float64)
+
+ for _, uuid := range uuids {
+ //csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+
+ if strings.Contains(uuid, "-d1hrv-") {
+ // This is a pipeline instance, not a job! Find the cwl-runner job.
+ pi := loadObject(logger, arv, resultsDir+"/"+uuid, uuid)
+ for _, v := range pi["components"].(map[string]interface{}) {
+ x := v.(map[string]interface{})
+ y := x["job"].(map[string]interface{})
+ uuid = y["uuid"].(string)
+ }
+ }
+
+ // for projects:
+ // arv -f uuid container_request list --filters '[["owner_uuid","=","<someuuid>"],["requesting_container_uuid","=",null]]'
+
+ if strings.Contains(uuid, "-j7d0g-") {
+ // This is a project (group)
+ for k, v := range handleProject(logger, uuid, arv, arv2, kc, resultsDir) {
+ cost[k] = v
+ }
+ } else if strings.Contains(uuid, "-xvhdp-") {
+ // This is a container request
+ for k, v := range generateCrCsv(logger, uuid, arv, arv2, kc, resultsDir) {
+ cost[k] = v
+ }
+ } else if strings.Contains(uuid, "-tpzed-") {
+ // This is a user. The "Home" project for a user is not a real project.
+ // It is identified by the user uuid. As such, cost analysis for the
+ // "Home" project is not supported by this program.
+ logger.Errorf("Cost analysis is not supported for the 'Home' project: %s", uuid)
+ }
+ }
+
+ logger.Info("\n")
+ for k := range cost {
+ logger.Infof("Uuid report in %s/%s.csv\n", resultsDir, k)
+ }
+
+ if len(cost) == 0 {
+ logger.Info("Nothing to do!\n")
+ os.Exit(0)
+ }
+
+ var csv string
+
+ csv = "# Aggregate cost accounting for uuids:\n"
+ for _, uuid := range uuids {
+ csv += "# " + uuid + "\n"
+ }
+
+ var total float64
+ for k, v := range cost {
+ csv += k + "," + strconv.FormatFloat(v, 'f', 8, 64) + "\n"
+ total += v
+ }
+
+ csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
+
+ // Write the resulting CSV file
+ aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+ err = ioutil.WriteFile(aFile, []byte(csv), 0644)
+ if err != nil {
+ logger.Errorf("Error writing file with path %s: %s\n", aFile, err.Error())
+ os.Exit(1)
+ } else {
+ logger.Infof("\nAggregate cost accounting for all supplied uuids in %s\n", aFile)
+ }
+ return
+}
commit 771e7c9387650565dbc87656d78e987ae43ba91b
Author: Ward Vandewege <ward at curii.com>
Date: Fri Oct 30 16:29:40 2020 -0400
16950: a few cleanups for our container request fixtures: attach a real
log collection, and define CompletedContainerRequestUUID in
arvadostest (go).
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 5677f4dec..9049c73c4 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -44,7 +44,8 @@ const (
RunningContainerUUID = "zzzzz-dz642-runningcontainr"
- CompletedContainerUUID = "zzzzz-dz642-compltcontainer"
+ CompletedContainerUUID = "zzzzz-dz642-compltcontainer"
+ CompletedContainerRequestUUID = "zzzzz-xvhdp-cr4completedctr"
ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
ArvadosRepoName = "arvados"
@@ -73,6 +74,8 @@ const (
TestVMUUID = "zzzzz-2x53u-382brsig8rp3064"
CollectionWithUniqueWordsUUID = "zzzzz-4zz18-mnt690klmb51aud"
+
+ LogCollectionUUID = "zzzzz-4zz18-logcollection01"
)
// PathologicalManifest : A valid manifest designed to test
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index a16ee8763..2243d6a44 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -1031,6 +1031,18 @@ collection_with_uri_prop:
properties:
"http://schema.org/example": "value1"
+log_collection:
+ uuid: zzzzz-4zz18-logcollection01
+ current_version_uuid: zzzzz-4zz18-logcollection01
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-10-29T00:51:44.075594000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-10-29T00:51:44.072109000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: a real log collection for a completed container
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
diff --git a/services/api/test/fixtures/container_requests.yml b/services/api/test/fixtures/container_requests.yml
index daaaf4d08..5816aa730 100644
--- a/services/api/test/fixtures/container_requests.yml
+++ b/services/api/test/fixtures/container_requests.yml
@@ -98,7 +98,7 @@ completed:
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-compltcontainer
- log_uuid: zzzzz-4zz18-y9vne9npefyxh8g
+ log_uuid: zzzzz-4zz18-logcollection01
output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
runtime_constraints:
vcpus: 1
@@ -324,7 +324,7 @@ completed_with_input_mounts:
vcpus: 1
ram: 123
container_uuid: zzzzz-dz642-compltcontainer
- log_uuid: zzzzz-4zz18-y9vne9npefyxh8g
+ log_uuid: zzzzz-4zz18-logcollection01
output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
mounts: {
"/var/lib/cwl/cwl.input.json": {
@@ -778,7 +778,7 @@ cr_in_trashed_project:
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-compltcontainer
- log_uuid: zzzzz-4zz18-y9vne9npefyxh8g
+ log_uuid: zzzzz-4zz18-logcollection01
output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
runtime_constraints:
vcpus: 1
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list