[ARVADOS] created: 1.3.0-1602-g591a25ea2
Git user
git at public.curoverse.com
Mon Sep 23 19:13:26 UTC 2019
at 591a25ea2d44801fbef2ec678a366807537a8411 (commit)
commit 591a25ea2d44801fbef2ec678a366807537a8411
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Sep 23 14:39:51 2019 -0400
15531: Adjust redirect_to_new_user to intended behavior
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/federation_migrate.py b/sdk/python/arvados/commands/federation_migrate.py
index ec846d2c4..3b3a7ee66 100755
--- a/sdk/python/arvados/commands/federation_migrate.py
+++ b/sdk/python/arvados/commands/federation_migrate.py
@@ -286,7 +286,7 @@ def main():
migratearv.users().merge(old_user_uuid=old_user_uuid,
new_user_uuid=new_user_uuid,
new_owner_uuid=grp["uuid"],
- redirect_to_new_user=old_user_uuid.startswith(migratecluster)).execute()
+ redirect_to_new_user=True).execute()
except arvados.errors.ApiError as e:
print("(%s) Error migrating user: %s" % (email, e))
diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index 0ef129481..2889eacee 100644
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -176,15 +176,10 @@ class Arvados::V1::UsersController < ApplicationController
return send_error("cannot move objects into supplied new_owner_uuid: new user does not have write permission", status: 403)
end
- redirect = params[:redirect_to_new_user]
- if @object.uuid[0..4] != Rails.configuration.ClusterID && redirect
- return send_error("cannot merge remote user to other with redirect_to_new_user=true", status: 422)
- end
-
act_as_system_user do
@object.merge(new_owner_uuid: params[:new_owner_uuid],
- redirect_to_user_uuid: new_user.uuid,
- redirect_auth: redirect)
+ new_user_uuid: new_user.uuid,
+ redirect_to_new_user: params[:redirect_to_new_user])
end
show
end
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index 59fb3fc09..08476be57 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -272,26 +272,28 @@ class User < ArvadosModel
end
end
- # Move this user's (i.e., self's) owned items into new_owner_uuid.
- # Also redirect future uses of this account to
- # redirect_to_user_uuid, i.e., when a caller authenticates to this
- # account in the future, the account redirect_to_user_uuid account
- # will be used instead.
+ # Move this user's (i.e., self's) owned items to new_owner_uuid and
+ # new_user_uuid (for things normally owned directly by the user).
+ #
+ # If redirect_auth is true, also reassign auth tokens and ssh keys,
+ # and redirect this account to redirect_to_user_uuid, i.e., when a
+ # caller authenticates to this account in the future, the account
+ # redirect_to_user_uuid account will be used instead.
#
# current_user must have admin privileges, i.e., the caller is
# responsible for checking permission to do this.
- def merge(new_owner_uuid:, redirect_to_user_uuid:, redirect_auth:)
+ def merge(new_owner_uuid:, new_user_uuid:, redirect_to_new_user:)
raise PermissionDeniedError if !current_user.andand.is_admin
- raise "not implemented" if !redirect_to_user_uuid
+ raise "not implemented" if !new_user_uuid
transaction(requires_new: true) do
reload
raise "cannot merge an already merged user" if self.redirect_to_user_uuid
- new_user = User.where(uuid: redirect_to_user_uuid).first
+ new_user = User.where(uuid: new_user_uuid).first
raise "user does not exist" if !new_user
raise "cannot merge to an already merged user" if new_user.redirect_to_user_uuid
- if redirect_auth
+ if redirect_to_new_user
# Existing API tokens and ssh keys are updated to authenticate
# to the new user.
ApiClientAuthorization.
@@ -313,8 +315,7 @@ class User < ArvadosModel
AuthorizedKey.where(authorized_user_uuid: uuid).destroy_all
user_updates = [
[Link, :owner_uuid],
- [Link, :tail_uuid],
- [Link, :head_uuid],
+ [Link, :tail_uuid]
]
end
@@ -351,7 +352,9 @@ class User < ArvadosModel
klass.where(owner_uuid: uuid).update_all(owner_uuid: new_owner_uuid)
end
- update_attributes!(redirect_to_user_uuid: new_user.uuid, username: nil)
+ if redirect_to_new_user
+ update_attributes!(redirect_to_user_uuid: new_user.uuid, username: nil)
+ end
invalidate_permissions_cache
end
end
diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb
index 4e19988de..f5c4ea0ef 100644
--- a/services/api/test/functional/arvados/v1/users_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/users_controller_test.rb
@@ -826,7 +826,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
redirect_to_new_user: false,
}
assert_response(:success)
- assert_equal(users(:active).uuid, User.unscoped.find_by_uuid(users(:project_viewer).uuid).redirect_to_user_uuid)
+ assert_nil(User.unscoped.find_by_uuid(users(:project_viewer).uuid).redirect_to_user_uuid)
# because redirect_to_new_user=false, token owned by
# project_viewer should be deleted
commit e0d49420ec4bff49f8fd1b79c1de708f2bea1aee
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Sep 23 11:32:03 2019 -0400
15531: Fed migrate script passes test cases
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/federation_migrate.py b/sdk/python/arvados/commands/federation_migrate.py
index 664a99765..ec846d2c4 100755
--- a/sdk/python/arvados/commands/federation_migrate.py
+++ b/sdk/python/arvados/commands/federation_migrate.py
@@ -116,8 +116,8 @@ def main():
homeuuid = ""
for a in accum:
r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
- by_email.setdefault(a["email"], [])
- by_email[a["email"]].append(r)
+ by_email.setdefault(a["email"], {})
+ by_email[a["email"]][a["uuid"]] = r
rows.append(r)
lastemail = u["email"]
accum = [u]
@@ -130,8 +130,8 @@ def main():
homeuuid = ""
for a in accum:
r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
- by_email.setdefault(a["email"], [])
- by_email[a["email"]].append(r)
+ by_email.setdefault(a["email"], {})
+ by_email[a["email"]][a["uuid"]] = r
rows.append(r)
if args.report:
@@ -147,13 +147,13 @@ def main():
print("Performing dry run")
rows = []
- by_email = {}
+
with open(args.migrate or args.dry_run, "rt") as f:
for r in csv.reader(f):
if r[0] == "email":
continue
- by_email.setdefault(r[0], [])
- by_email[r[0]].append(r)
+ by_email.setdefault(r[0], {})
+ by_email[r[0]][r[2]] = r
rows.append(r)
for r in rows:
@@ -165,10 +165,22 @@ def main():
if userhome == "":
print("(%s) Skipping %s, no home cluster specified" % (email, old_user_uuid))
if old_user_uuid.startswith(userhome):
+ migratecluster = old_user_uuid[0:5]
+ migratearv = clusters[migratecluster]
+ if migratearv.users().get(uuid=old_user_uuid).execute()["username"] != username:
+ print("(%s) Updating username of %s to '%s' on %s" % (email, old_user_uuid, username, migratecluster))
+ if not args.dry_run:
+ try:
+ conflicts = migratearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ migratearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
+ migratearv.users().update(uuid=old_user_uuid, body={"user": {"username": username}}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, old_user_uuid, username, migratecluster, e))
continue
candidates = []
conflict = False
- for b in by_email[email]:
+ for b in by_email[email].values():
if b[2].startswith(userhome):
candidates.append(b)
if b[1] != username and b[3] == userhome:
@@ -196,13 +208,11 @@ def main():
continue
tup = (email, username, user["uuid"], userhome)
- by_email[email].append(tup)
- candidates.append(tup)
else:
# dry run
tup = (email, username, "%s-tpzed-xfakexfakexfake" % (userhome[0:5]), userhome)
- by_email[email].append(tup)
- candidates.append(tup)
+ by_email[email][tup[2]] = tup
+ candidates.append(tup)
if len(candidates) > 1:
print("(%s) Multiple users listed to migrate %s to %s, use full uuid" % (email, old_user_uuid, userhome))
continue
@@ -282,10 +292,11 @@ def main():
if newuser['username'] != username:
try:
- conflicts = migratearv.users().list(filters=[["username", "=", username]]).execute()
- if conflicts["items"]:
- migratearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
- migratearv.users().update(uuid=new_user_uuid, body={"user": {"username": username}}).execute()
+ if not args.dry_run:
+ conflicts = migratearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ migratearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
+ migratearv.users().update(uuid=new_user_uuid, body={"user": {"username": username}}).execute()
except arvados.errors.ApiError as e:
print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, new_user_uuid, username, migratecluster, e))
diff --git a/sdk/python/tests/fed-migrate/check.py b/sdk/python/tests/fed-migrate/check.py
new file mode 100644
index 000000000..3927954ce
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/check.py
@@ -0,0 +1,45 @@
+import arvados
+import json
+import sys
+
+j = json.load(open(sys.argv[1]))
+
+apiA = arvados.api(host=j["arvados_api_hosts"][0], token=j["superuser_tokens"][0], insecure=True)
+apiB = arvados.api(host=j["arvados_api_hosts"][1], token=j["superuser_tokens"][1], insecure=True)
+apiC = arvados.api(host=j["arvados_api_hosts"][2], token=j["superuser_tokens"][2], insecure=True)
+
+users = apiA.users().list().execute()
+
+assert len(users["items"]) == 10
+
+by_username = {}
+
+for i in range(1, 9):
+ found = False
+ for u in users["items"]:
+ if u["username"] == ("case%d" % i) and u["email"] == ("case%d at test" % i):
+ found = True
+ by_username[u["username"]] = u["uuid"]
+ assert found
+
+users = apiB.users().list().execute()
+assert len(users["items"]) == 10
+
+for i in range(2, 9):
+ found = False
+ for u in users["items"]:
+ if u["username"] == ("case%d" % i) and u["email"] == ("case%d at test" % i) and u["uuid"] == by_username[u["username"]]:
+ found = True
+ assert found
+
+users = apiC.users().list().execute()
+assert len(users["items"]) == 10
+
+for i in range(2, 9):
+ found = False
+ for u in users["items"]:
+ if u["username"] == ("case%d" % i) and u["email"] == ("case%d at test" % i) and u["uuid"] == by_username[u["username"]]:
+ found = True
+ assert found
+
+print("Passed checks")
diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwl b/sdk/python/tests/fed-migrate/fed-migrate.cwl
index a94dfb5b6..313946dd3 100644
--- a/sdk/python/tests/fed-migrate/fed-migrate.cwl
+++ b/sdk/python/tests/fed-migrate/fed-migrate.cwl
@@ -16,8 +16,8 @@ $graph:
id: fed_migrate
type: string
outputs:
- - id: report
- outputSource: main_2/report
+ - id: report3
+ outputSource: main_2/report3
type: File
requirements:
InlineJavascriptRequirement: {}
@@ -89,23 +89,22 @@ $graph:
valueFrom: '$(inputs.superuser_tokens[0])'
out:
- report
+ - report2
+ - report3
+ - r
run:
- arguments:
- - $(inputs.fed_migrate)
- - '--report'
- - report.csv
- class: CommandLineTool
+ class: Workflow
id: main_2_embed
inputs:
- - id: arvados_api_hosts
+ - id: ar
type:
items: string
type: array
- - id: superuser_tokens
+ - id: arvados_api_hosts
type:
items: string
type: array
- - id: ar
+ - id: superuser_tokens
type:
items: string
type: array
@@ -117,14 +116,179 @@ $graph:
type: Any
outputs:
- id: report
- outputBinding:
- glob: report.csv
+ outputSource: main_2_embed_1/report
+ type: File
+ - id: report2
+ outputSource: main_2_embed_2/report2
+ type: File
+ - id: report3
+ outputSource: main_2_embed_3/report3
+ type: File
+ - id: r
+ outputSource: main_2_embed_4/r
type: File
requirements:
- class: EnvVarRequirement
envDef:
ARVADOS_API_HOST: $(inputs.host)
ARVADOS_API_TOKEN: $(inputs.token)
+ steps:
+ - id: main_2_embed_1
+ in:
+ fed_migrate:
+ source: fed_migrate
+ host:
+ source: host
+ token:
+ source: token
+ out:
+ - report
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--report'
+ - report.csv
+ class: CommandLineTool
+ id: main_2_embed_1_embed
+ inputs:
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report
+ outputBinding:
+ glob: report.csv
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_2
+ in:
+ host:
+ source: host
+ report:
+ source: main_2_embed_1/report
+ token:
+ source: token
+ out:
+ - report2
+ run:
+ arguments:
+ - sed
+ - '-E'
+ - 's/,(case[1-8])2?,/,\1,/g'
+ class: CommandLineTool
+ id: main_2_embed_2_embed
+ inputs:
+ - id: report
+ type: File
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report2
+ outputBinding:
+ glob: report.csv
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ stdin: $(inputs.report.path)
+ stdout: report.csv
+ - id: main_2_embed_3
+ in:
+ fed_migrate:
+ source: fed_migrate
+ host:
+ source: host
+ report2:
+ source: main_2_embed_2/report2
+ token:
+ source: token
+ out:
+ - report3
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--migrate'
+ - $(inputs.report2)
+ class: CommandLineTool
+ id: main_2_embed_3_embed
+ inputs:
+ - id: report2
+ type: File
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report3
+ outputBinding:
+ outputEval: $(inputs.report2)
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_4
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ check:
+ default:
+ class: File
+ location: check.py
+ host:
+ source: host
+ report3:
+ source: main_2_embed_3/report3
+ superuser_tokens:
+ source: superuser_tokens
+ token:
+ source: token
+ out:
+ - r
+ run:
+ arguments:
+ - python
+ - $(inputs.check)
+ - _script
+ class: CommandLineTool
+ id: main_2_embed_4_embed
+ inputs:
+ - id: report3
+ type: File
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: check
+ type: File
+ outputs:
+ - id: r
+ outputBinding:
+ outputEval: $(inputs.report3)
+ type: File
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: |
+ {
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+ }
+ entryname: _script
+ InlineJavascriptRequirement: {}
- arguments:
- arvbox
- cat
@@ -177,7 +341,7 @@ $graph:
items: string
type: array
- id: report
- outputSource: run_test_3/report
+ outputSource: run_test_3/report3
type: File
requirements:
InlineJavascriptRequirement: {}
@@ -368,7 +532,7 @@ $graph:
superuser_tokens:
source: main_2/supertok
out:
- - report
+ - report3
run: '#run_test'
cwlVersion: v1.0
diff --git a/sdk/python/tests/fed-migrate/run-test.cwl b/sdk/python/tests/fed-migrate/run-test.cwl
index ea412ac8e..623a9c11f 100644
--- a/sdk/python/tests/fed-migrate/run-test.cwl
+++ b/sdk/python/tests/fed-migrate/run-test.cwl
@@ -15,8 +15,8 @@ inputs:
id: fed_migrate
type: string
outputs:
- - id: out
- outputSource: main_2/out
+ - id: report3
+ outputSource: main_2/report3
type: File
requirements:
InlineJavascriptRequirement: {}
@@ -87,24 +87,23 @@ steps:
token:
valueFrom: '$(inputs.superuser_tokens[0])'
out:
- - out
+ - report
+ - report2
+ - report3
+ - r
run:
- arguments:
- - $(inputs.fed_migrate)
- - '--report'
- - out
- class: CommandLineTool
+ class: Workflow
id: main_2_embed
inputs:
- - id: arvados_api_hosts
+ - id: ar
type:
items: string
type: array
- - id: superuser_tokens
+ - id: arvados_api_hosts
type:
items: string
type: array
- - id: ar
+ - id: superuser_tokens
type:
items: string
type: array
@@ -115,13 +114,178 @@ steps:
- id: token
type: Any
outputs:
- - id: out
- outputBinding:
- glob: out
+ - id: report
+ outputSource: main_2_embed_1/report
+ type: File
+ - id: report2
+ outputSource: main_2_embed_2/report2
+ type: File
+ - id: report3
+ outputSource: main_2_embed_3/report3
+ type: File
+ - id: r
+ outputSource: main_2_embed_4/r
type: File
requirements:
- class: EnvVarRequirement
envDef:
ARVADOS_API_HOST: $(inputs.host)
ARVADOS_API_TOKEN: $(inputs.token)
+ steps:
+ - id: main_2_embed_1
+ in:
+ fed_migrate:
+ source: fed_migrate
+ host:
+ source: host
+ token:
+ source: token
+ out:
+ - report
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--report'
+ - report.csv
+ class: CommandLineTool
+ id: main_2_embed_1_embed
+ inputs:
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report
+ outputBinding:
+ glob: report.csv
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_2
+ in:
+ host:
+ source: host
+ report:
+ source: main_2_embed_1/report
+ token:
+ source: token
+ out:
+ - report2
+ run:
+ arguments:
+ - sed
+ - '-E'
+ - 's/,(case[1-8])2?,/,1,/g'
+ class: CommandLineTool
+ id: main_2_embed_2_embed
+ inputs:
+ - id: report
+ type: File
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report2
+ outputBinding:
+ glob: report.csv
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ stdin: $(inputs.report)
+ stdout: report.csv
+ - id: main_2_embed_3
+ in:
+ fed_migrate:
+ source: fed_migrate
+ host:
+ source: host
+ report2:
+ source: main_2_embed_2/report2
+ token:
+ source: token
+ out:
+ - report3
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--migrate'
+ - $(inputs.report)
+ class: CommandLineTool
+ id: main_2_embed_3_embed
+ inputs:
+ - id: report2
+ type: File
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report3
+ outputBinding:
+ outputEval: $(inputs.report2)
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_4
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ check:
+ default:
+ class: File
+ location: check.py
+ host:
+ source: host
+ report3:
+ source: main_2_embed_3/report3
+ superuser_tokens:
+ source: superuser_tokens
+ token:
+ source: token
+ out:
+ - r
+ run:
+ arguments:
+ - python
+ - $(inputs.check)
+ - _script
+ class: CommandLineTool
+ id: main_2_embed_4_embed
+ inputs:
+ - id: report3
+ type: File
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: check
+ type: File
+ outputs:
+ - id: r
+ outputBinding:
+ outputEval: $(inputs.report3)
+ type: File
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: |
+ {
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+ }
+ entryname: _script
+ InlineJavascriptRequirement: {}
diff --git a/sdk/python/tests/fed-migrate/run-test.cwlex b/sdk/python/tests/fed-migrate/run-test.cwlex
index 3dda1fe7f..ef37c5152 100644
--- a/sdk/python/tests/fed-migrate/run-test.cwlex
+++ b/sdk/python/tests/fed-migrate/run-test.cwlex
@@ -11,13 +11,13 @@ def workflow main(
"superuser_tokens": $(inputs.superuser_tokens)
}
>>>
- return arvados_api_hosts as ar
+ return arvados_api_hosts as ar
}
- run tool(arvados_api_hosts, superuser_tokens, ar,
- fed_migrate,
- host=$(inputs.arvados_api_hosts[0]),
- token=$(inputs.superuser_tokens[0])) {
+ run workflow(ar, arvados_api_hosts, superuser_tokens,
+ fed_migrate,
+ host=$(inputs.arvados_api_hosts[0]),
+ token=$(inputs.superuser_tokens[0])) {
requirements {
EnvVarRequirement {
envDef: {
@@ -26,9 +26,32 @@ def workflow main(
}
}
}
- $(inputs.fed_migrate) --report report.csv
- return File("report.csv") as report
+
+ run tool(fed_migrate, host, token) {
+ $(inputs.fed_migrate) --report report.csv
+ return File("report.csv") as report
+ }
+
+ run tool(report, host, token) {
+ sed -E 's/,(case[1-8])2?,/,\\1,/g' < $(inputs.report.path) > report.csv
+ return File("report.csv") as report2
+ }
+
+ run tool(report2, fed_migrate, host, token) {
+ $(inputs.fed_migrate) --migrate $(inputs.report2)
+ return report2 as report3
+ }
+
+ run tool(report3, host, token, arvados_api_hosts, superuser_tokens, check=File("check.py")) {
+ python $(inputs.check) <<<
+{
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+}
+>>>
+ return report3 as r
+ }
}
- return report
+ return report3
}
\ No newline at end of file
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index e56fbd489..2999d3193 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -242,6 +242,8 @@ run() {
fi
if ! test -d "$COMPOSER_ROOT" ; then
git clone https://github.com/curoverse/composer.git "$COMPOSER_ROOT"
+ git -C "$COMPOSER_ROOT" checkout arvados-fork
+ git -C "$COMPOSER_ROOT" pull
fi
if ! test -d "$WORKBENCH2_ROOT" ; then
git clone https://github.com/curoverse/arvados-workbench2.git "$WORKBENCH2_ROOT"
@@ -594,6 +596,39 @@ case "$subcmd" in
exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados'
;;
+ checkpoint)
+ exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > /var/lib/arvados/checkpoint.sql'
+ ;;
+
+ restore)
+ exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=/var/lib/arvados/checkpoint.sql'
+ ;;
+
+ hotreset)
+ exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash - <<EOF
+sv stop api
+sv stop controller
+sv stop websockets
+sv stop keepstore0
+sv stop keepstore1
+sv stop keepproxy
+cd /usr/src/arvados/services/api
+export RAILS_ENV=development
+bundle exec rake db:drop
+rm /var/lib/arvados/api_database_setup
+rm /var/lib/arvados/superuser_token
+rm /var/lib/arvados/keep0-uuid
+rm /var/lib/arvados/keep1-uuid
+rm /var/lib/arvados/keepproxy-uuid
+sv start api
+sv start controller
+sv start websockets
+sv restart keepstore0
+sv restart keepstore1
+sv restart keepproxy
+EOF
+ ;;
+
*)
echo "Arvados-in-a-box https://doc.arvados.org/install/arvbox.html"
echo
@@ -612,6 +647,8 @@ case "$subcmd" in
echo "build <config> build arvbox Docker image"
echo "reboot <config> stop, build arvbox Docker image, run"
echo "rebuild <config> build arvbox Docker image, no layer cache"
+ echo "checkpoint create database backup"
+ echo "restore restore checkpoint"
echo "reset delete arvbox arvados data (be careful!)"
echo "destroy delete all arvbox code and data (be careful!)"
echo "log <service> tail log of specified service"
diff --git a/tools/arvbox/lib/arvbox/docker/cluster-config.sh b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
index 951b592ea..89d1a4807 100755
--- a/tools/arvbox/lib/arvbox/docker/cluster-config.sh
+++ b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
@@ -6,7 +6,7 @@
exec 2>&1
set -ex -o pipefail
-if [[ -s /etc/arvados/config.yml ]] ; then
+if [[ -s /etc/arvados/config.yml ]] && [[ /var/lib/arvados/cluster_config.yml.override -ot /etc/arvados/config.yml ]] ; then
exit
fi
@@ -82,7 +82,7 @@ Clusters:
Keepproxy:
InternalURLs:
"http://localhost:${services[keepproxy]}/": {}
- ExternalURL: "http://$localip:${services[keepproxy-ssl]}/"
+ ExternalURL: "https://$localip:${services[keepproxy-ssl]}/"
Websocket:
ExternalURL: "wss://$localip:${services[websockets-ssl]}/websocket"
InternalURLs:
@@ -104,7 +104,7 @@ Clusters:
InternalURLs:
"http://localhost:${services[keep-web]}/": {}
Composer:
- ExternalURL: "http://$localip:${services[composer]}"
+ ExternalURL: "https://$localip:${services[composer]}"
Controller:
ExternalURL: "https://$localip:${services[controller-ssl]}"
InternalURLs:
commit 0f537bcaa60b8a1496010bc9d4a943484e69081c
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Sep 20 13:59:46 2019 -0400
15531: Test federation migrate script WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/tests/federation/arvbox-make-federation.cwl b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
index 9a08195a7..341ce1228 100644
--- a/sdk/cwl/tests/federation/arvbox-make-federation.cwl
+++ b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
@@ -32,12 +32,18 @@ outputs:
arvados_cluster_ids:
type: string[]
outputSource: start/cluster_id
+ superuser_tokens:
+ type: string[]
+ outputSource: start/superuser_token
acr:
type: string?
outputSource: in_acr
arvado_api_host_insecure:
type: boolean
outputSource: insecure
+ arvbox_containers:
+ type: string[]
+ outputSource: containers
steps:
mkdir:
in:
diff --git a/sdk/cwl/tests/federation/arvbox/setup-user.cwl b/sdk/cwl/tests/federation/arvbox/setup-user.cwl
index 0fddc1b87..a3ad6e575 100644
--- a/sdk/cwl/tests/federation/arvbox/setup-user.cwl
+++ b/sdk/cwl/tests/federation/arvbox/setup-user.cwl
@@ -31,4 +31,4 @@ inputs:
outputs:
test_user_uuid: string
test_user_token: string
-arguments: [python2, $(inputs.make_user_script)]
\ No newline at end of file
+arguments: [python, $(inputs.make_user_script)]
diff --git a/sdk/python/arvados/commands/federation_migrate.py b/sdk/python/arvados/commands/federation_migrate.py
index 386c0ef9b..664a99765 100755
--- a/sdk/python/arvados/commands/federation_migrate.py
+++ b/sdk/python/arvados/commands/federation_migrate.py
@@ -39,8 +39,11 @@ def main():
else:
arv = arvados.api(cache=False)
rh = arv._rootDesc["remoteHosts"]
+ tok = arv.api_client_authorizations().current().execute()
+ token = "v2/%s/%s" % (tok["uuid"], tok["api_token"])
+
for k,v in rh.items():
- arv = arvados.api(host=v, token=os.environ["ARVADOS_API_TOKEN"], cache=False)
+ arv = arvados.api(host=v, token=token, cache=False, insecure=os.environ.get("ARVADOS_API_HOST_INSECURE"))
config = arv.configs().get().execute()
if config["Login"]["LoginCluster"] != "" and loginCluster is None:
loginCluster = config["Login"]["LoginCluster"]
@@ -82,52 +85,62 @@ def main():
print("Tokens file passed checks")
exit(0)
- if args.report:
- users = []
- for c, arv in clusters.items():
- print("Getting user list from %s" % c)
- ul = arvados.util.list_all(arv.users().list)
- for l in ul:
- if l["uuid"].startswith(c):
- users.append(l)
+ rows = []
+ by_email = {}
- out = csv.writer(open(args.report, "wt"))
+ users = []
+ for c, arv in clusters.items():
+ print("Getting user list from %s" % c)
+ ul = arvados.util.list_all(arv.users().list)
+ for l in ul:
+ if l["uuid"].startswith(c):
+ users.append(l)
- out.writerow(("email", "username", "user uuid", "home cluster"))
+ users = sorted(users, key=lambda u: u["email"]+"::"+(u["username"] or "")+"::"+u["uuid"])
- users = sorted(users, key=lambda u: u["email"]+"::"+(u["username"] or "")+"::"+u["uuid"])
+ accum = []
+ lastemail = None
+ for u in users:
+ if u["uuid"].endswith("-anonymouspublic") or u["uuid"].endswith("-000000000000000"):
+ continue
+ if lastemail == None:
+ lastemail = u["email"]
+ if u["email"] == lastemail:
+ accum.append(u)
+ else:
+ homeuuid = None
+ for a in accum:
+ if homeuuid is None:
+ homeuuid = a["uuid"]
+ if a["uuid"] != homeuuid:
+ homeuuid = ""
+ for a in accum:
+ r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
+ by_email.setdefault(a["email"], [])
+ by_email[a["email"]].append(r)
+ rows.append(r)
+ lastemail = u["email"]
+ accum = [u]
- accum = []
- lastemail = None
- for u in users:
- if u["uuid"].endswith("-anonymouspublic") or u["uuid"].endswith("-000000000000000"):
- continue
- if lastemail == None:
- lastemail = u["email"]
- if u["email"] == lastemail:
- accum.append(u)
- else:
- homeuuid = None
- for a in accum:
- if homeuuid is None:
- homeuuid = a["uuid"]
- if a["uuid"] != homeuuid:
- homeuuid = ""
- for a in accum:
- out.writerow((a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5]))
- lastemail = u["email"]
- accum = [u]
-
- homeuuid = None
- for a in accum:
- if homeuuid is None:
- homeuuid = a["uuid"]
- if a["uuid"] != homeuuid:
- homeuuid = ""
- for a in accum:
- out.writerow((a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5]))
+ homeuuid = None
+ for a in accum:
+ if homeuuid is None:
+ homeuuid = a["uuid"]
+ if a["uuid"] != homeuuid:
+ homeuuid = ""
+ for a in accum:
+ r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
+ by_email.setdefault(a["email"], [])
+ by_email[a["email"]].append(r)
+ rows.append(r)
+ if args.report:
+ out = csv.writer(open(args.report, "wt"))
+ out.writerow(("email", "username", "user uuid", "home cluster"))
+ for r in rows:
+ out.writerow(r)
print("Wrote %s" % args.report)
+ return
if args.migrate or args.dry_run:
if args.dry_run:
@@ -142,6 +155,7 @@ def main():
by_email.setdefault(r[0], [])
by_email[r[0]].append(r)
rows.append(r)
+
for r in rows:
email = r[0]
username = r[1]
@@ -153,9 +167,16 @@ def main():
if old_user_uuid.startswith(userhome):
continue
candidates = []
+ conflict = False
for b in by_email[email]:
if b[2].startswith(userhome):
candidates.append(b)
+ if b[1] != username and b[3] == userhome:
+ print("(%s) Cannot migrate %s, conflicting usernames %s and %s" % (email, old_user_uuid, b[1], username))
+ conflict = True
+ break
+ if conflict:
+ continue
if len(candidates) == 0:
if len(userhome) == 5 and userhome not in clusters:
print("(%s) Cannot migrate %s, unknown home cluster %s (typo?)" % (email, old_user_uuid, userhome))
@@ -166,21 +187,22 @@ def main():
homearv = clusters[userhome]
user = None
try:
+ conflicts = homearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ homearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
user = homearv.users().create(body={"user": {"email": email, "username": username}}).execute()
except arvados.errors.ApiError as e:
- if "Username" in str(e):
- other = homearv.users().list(filters=[["username", "=", username]]).execute()
- if other['items'] and other['items'][0]['email'] == email:
- conflicting_user = other['items'][0]
- homearv.users().update(uuid=conflicting_user["uuid"], body={"user": {"username": username+"migrate"}}).execute()
- user = homearv.users().create(body={"user": {"email": email, "username": username}}).execute()
- if not user:
- print("(%s) Could not create user: %s" % (email, str(e)))
- continue
-
- candidates.append((email, username, user["uuid"], userhome))
+ print("(%s) Could not create user: %s" % (email, str(e)))
+ continue
+
+ tup = (email, username, user["uuid"], userhome)
+ by_email[email].append(tup)
+ candidates.append(tup)
else:
- candidates.append((email, username, "%s-tpzed-xfakexfakexfake" % (userhome[0:5]), userhome))
+ # dry run
+ tup = (email, username, "%s-tpzed-xfakexfakexfake" % (userhome[0:5]), userhome)
+ by_email[email].append(tup)
+ candidates.append(tup)
if len(candidates) > 1:
print("(%s) Multiple users listed to migrate %s to %s, use full uuid" % (email, old_user_uuid, userhome))
continue
@@ -215,9 +237,9 @@ def main():
try:
ru = urllib.parse.urlparse(migratearv._rootDesc["rootUrl"])
if not args.dry_run:
- newuser = arvados.api(host=ru.netloc, token=salted).users().current().execute()
+ newuser = arvados.api(host=ru.netloc, token=salted, insecure=os.environ.get("ARVADOS_API_HOST_INSECURE")).users().current().execute()
else:
- newuser = {"is_active": True}
+ newuser = {"is_active": True, "username": username}
except arvados.errors.ApiError as e:
print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
continue
@@ -259,8 +281,10 @@ def main():
print("(%s) Error migrating user: %s" % (email, e))
if newuser['username'] != username:
- print("%s != %s" % (newuser['username'], username))
try:
+ conflicts = migratearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ migratearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
migratearv.users().update(uuid=new_user_uuid, body={"user": {"username": username}}).execute()
except arvados.errors.ApiError as e:
print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, new_user_uuid, username, migratecluster, e))
diff --git a/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl b/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
new file mode 100644
index 000000000..c3fcbdcb3
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
@@ -0,0 +1,30 @@
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ arvbox_base: Directory
+outputs:
+ arvados_api_hosts:
+ type: string[]
+ outputSource: start/arvados_api_hosts
+ arvados_cluster_ids:
+ type: string[]
+ outputSource: start/arvados_cluster_ids
+ superuser_tokens:
+ type: string[]
+ outputSource: start/superuser_tokens
+ arvbox_containers:
+ type: string[]
+ outputSource: start/arvbox_containers
+requirements:
+ SubworkflowFeatureRequirement: {}
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+steps:
+ start:
+ in:
+ arvbox_base: arvbox_base
+ out: [arvados_api_hosts, arvados_cluster_ids, arvado_api_host_insecure, superuser_tokens, arvbox_containers]
+ run: ../../../cwl/tests/federation/arvbox-make-federation.cwl
diff --git a/sdk/python/tests/fed-migrate/create_users.py b/sdk/python/tests/fed-migrate/create_users.py
new file mode 100644
index 000000000..08dec5cde
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/create_users.py
@@ -0,0 +1,84 @@
+import arvados
+import json
+import sys
+
+j = json.load(open(sys.argv[1]))
+
+apiA = arvados.api(host=j["arvados_api_hosts"][0], token=j["superuser_tokens"][0], insecure=True)
+apiB = arvados.api(host=j["arvados_api_hosts"][1], token=j["superuser_tokens"][1], insecure=True)
+apiC = arvados.api(host=j["arvados_api_hosts"][2], token=j["superuser_tokens"][2], insecure=True)
+
+def maketoken(newtok):
+ return 'v2/' + newtok["uuid"] + '/' + newtok["api_token"]
+
+# case 1
+# user only exists on cluster A
+apiA.users().create(body={"user": {"email": "case1 at test"}}).execute()
+
+# case 2
+# user exists on cluster A and has remotes on B and C
+case2 = apiA.users().create(body={"user": {"email": "case2 at test"}}).execute()
+newtok = apiA.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case2["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtok), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtok), insecure=True).users().current().execute()
+
+# case 3
+# user only exists on cluster B
+case3 = apiB.users().create(body={"user": {"email": "case3 at test"}}).execute()
+
+# case 4
+# user only exists on cluster B and has remotes on A and C
+case4 = apiB.users().create(body={"user": {"email": "case4 at test"}}).execute()
+newtok = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case4["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtok), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtok), insecure=True).users().current().execute()
+
+
+# case 5
+# user exists on both cluster A and B
+case5 = apiA.users().create(body={"user": {"email": "case5 at test"}}).execute()
+case5 = apiB.users().create(body={"user": {"email": "case5 at test"}}).execute()
+
+# case 6
+# user exists on both cluster A and B, with remotes on A, B and C
+case6_A = apiA.users().create(body={"user": {"email": "case6 at test"}}).execute()
+newtokA = apiA.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case6_A["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokA), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokA), insecure=True).users().current().execute()
+
+case6_B = apiB.users().create(body={"user": {"email": "case6 at test"}}).execute()
+newtokB = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case6_B["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
+
+# case 7
+# user exists on both cluster B and A, with remotes on A, B and C
+case7_B = apiB.users().create(body={"user": {"email": "case7 at test"}}).execute()
+newtokB = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case7_B["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
+
+case7_A = apiA.users().create(body={"user": {"email": "case7 at test"}}).execute()
+newtokA = apiA.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case7_A["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokA), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokA), insecure=True).users().current().execute()
+
+# case 8
+# user exists on both cluster B and C, with remotes on A, B and C
+case8_B = apiB.users().create(body={"user": {"email": "case8 at test"}}).execute()
+newtokB = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case8_B["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
+
+case8_C = apiC.users().create(body={"user": {"email": "case8 at test"}}).execute()
+newtokC = apiC.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case8_C["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokC), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokC), insecure=True).users().current().execute()
diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwl b/sdk/python/tests/fed-migrate/fed-migrate.cwl
new file mode 100644
index 000000000..a94dfb5b6
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/fed-migrate.cwl
@@ -0,0 +1,374 @@
+#!/usr/bin/env cwl-runner
+$graph:
+ - class: Workflow
+ cwlVersion: v1.0
+ id: '#run_test'
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - default: arv-federation-migrate
+ id: fed_migrate
+ type: string
+ outputs:
+ - id: report
+ outputSource: main_2/report
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ MultipleInputFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+ SubworkflowFeatureRequirement: {}
+ steps:
+ - id: main_1
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ create_users:
+ default:
+ class: File
+ location: create_users.py
+ superuser_tokens:
+ source: superuser_tokens
+ out:
+ - ar
+ run:
+ arguments:
+ - python
+ - $(inputs.create_users)
+ - _script
+ class: CommandLineTool
+ id: main_1_embed
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: create_users
+ type: File
+ outputs:
+ - id: ar
+ outputBinding:
+ outputEval: $(inputs.arvados_api_hosts)
+ type:
+ items: string
+ type: array
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: |
+ {
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+ }
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: main_2
+ in:
+ ar:
+ source: main_1/ar
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ fed_migrate:
+ source: fed_migrate
+ host:
+ valueFrom: '$(inputs.arvados_api_hosts[0])'
+ superuser_tokens:
+ source: superuser_tokens
+ token:
+ valueFrom: '$(inputs.superuser_tokens[0])'
+ out:
+ - report
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--report'
+ - report.csv
+ class: CommandLineTool
+ id: main_2_embed
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: ar
+ type:
+ items: string
+ type: array
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report
+ outputBinding:
+ glob: report.csv
+ type: File
+ requirements:
+ - class: EnvVarRequirement
+ envDef:
+ ARVADOS_API_HOST: $(inputs.host)
+ ARVADOS_API_TOKEN: $(inputs.token)
+ - arguments:
+ - arvbox
+ - cat
+ - /var/lib/arvados/superuser_token
+ class: CommandLineTool
+ cwlVersion: v1.0
+ id: '#superuser_tok'
+ inputs:
+ - id: container
+ type: string
+ outputs:
+ - id: superuser_token
+ outputBinding:
+ glob: superuser_token.txt
+ loadContents: true
+ outputEval: '$(self[0].contents.trim())'
+ type: string
+ requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container)
+ InlineJavascriptRequirement: {}
+ stdout: superuser_token.txt
+ - class: Workflow
+ id: '#main'
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: arvados_cluster_ids
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: arvbox_containers
+ type:
+ items: string
+ type: array
+ - default: arv-federation-migrate
+ id: fed_migrate
+ type: string
+ outputs:
+ - id: supertok
+ outputSource: main_2/supertok
+ type:
+ items: string
+ type: array
+ - id: report
+ outputSource: run_test_3/report
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ MultipleInputFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+ SubworkflowFeatureRequirement: {}
+ steps:
+ - id: main_1
+ in:
+ arvados_cluster_ids:
+ source: arvados_cluster_ids
+ out:
+ - logincluster
+ run:
+ class: ExpressionTool
+ expression: '${return {''logincluster'': (inputs.arvados_cluster_ids[0])};}'
+ inputs:
+ - id: arvados_cluster_ids
+ type:
+ items: string
+ type: array
+ outputs:
+ - id: logincluster
+ type: string
+ - id: main_2
+ in:
+ cluster_id:
+ source: arvados_cluster_ids
+ container:
+ source: arvbox_containers
+ host:
+ source: arvados_api_hosts
+ logincluster:
+ source: main_1/logincluster
+ out:
+ - supertok
+ run:
+ class: Workflow
+ id: main_2_embed
+ inputs:
+ - id: container
+ type: string
+ - id: cluster_id
+ type: string
+ - id: host
+ type: string
+ - id: logincluster
+ type: string
+ outputs:
+ - id: supertok
+ outputSource: superuser_tok_3/superuser_token
+ type: string
+ requirements:
+ - class: EnvVarRequirement
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container)
+ steps:
+ - id: main_2_embed_1
+ in:
+ cluster_id:
+ source: cluster_id
+ container:
+ source: container
+ logincluster:
+ source: logincluster
+ set_login:
+ default:
+ class: File
+ location: set_login.py
+ out:
+ - c
+ run:
+ arguments:
+ - sh
+ - _script
+ class: CommandLineTool
+ id: main_2_embed_1_embed
+ inputs:
+ - id: container
+ type: string
+ - id: cluster_id
+ type: string
+ - id: logincluster
+ type: string
+ - id: set_login
+ type: File
+ outputs:
+ - id: c
+ outputBinding:
+ outputEval: $(inputs.container)
+ type: string
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: >
+ set -x
+
+ docker cp
+ $(inputs.container):/var/lib/arvados/cluster_config.yml.override
+ .
+
+ chmod +w cluster_config.yml.override
+
+ python $(inputs.set_login.path)
+ cluster_config.yml.override $(inputs.cluster_id)
+ $(inputs.logincluster)
+
+ docker cp cluster_config.yml.override
+ $(inputs.container):/var/lib/arvados
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_2
+ in:
+ c:
+ source: main_2_embed_1/c
+ container:
+ source: container
+ host:
+ source: host
+ out:
+ - d
+ run:
+ arguments:
+ - sh
+ - _script
+ class: CommandLineTool
+ id: main_2_embed_2_embed
+ inputs:
+ - id: container
+ type: string
+ - id: host
+ type: string
+ - id: c
+ type: string
+ outputs:
+ - id: d
+ outputBinding:
+ outputEval: $(inputs.c)
+ type: string
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: >
+ set -x
+
+ arvbox hotreset
+
+ while ! curl --fail --insecure --silent
+ https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest
+ >/dev/null ; do sleep 3 ; done
+
+ export ARVADOS_API_HOST=$(inputs.host)
+
+ export ARVADOS_API_TOKEN=\$(arvbox cat
+ /var/lib/arvados/superuser_token)
+
+ export ARVADOS_API_HOST_INSECURE=1
+
+ ARVADOS_VIRTUAL_MACHINE_UUID=\$(arvbox cat
+ /var/lib/arvados/vm-uuid)
+
+ while ! python -c "import arvados ;
+ arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()"
+ 2>/dev/null ; do sleep 3; done
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: superuser_tok_3
+ in:
+ container:
+ source: container
+ d:
+ source: main_2_embed_2/d
+ out:
+ - superuser_token
+ run: '#superuser_tok'
+ scatter:
+ - container
+ - cluster_id
+ - host
+ scatterMethod: dotproduct
+ - id: run_test_3
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ fed_migrate:
+ source: fed_migrate
+ superuser_tokens:
+ source: main_2/supertok
+ out:
+ - report
+ run: '#run_test'
+cwlVersion: v1.0
+
diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwlex b/sdk/python/tests/fed-migrate/fed-migrate.cwlex
new file mode 100644
index 000000000..c39093807
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/fed-migrate.cwlex
@@ -0,0 +1,56 @@
+import "run-test.cwlex" as run_test
+import "superuser-tok.cwl" as superuser_tok
+
+def workflow main(
+ arvados_api_hosts string[],
+ arvados_cluster_ids string[],
+ superuser_tokens string[],
+ arvbox_containers string[],
+ fed_migrate="arv-federation-migrate"
+) {
+
+ logincluster = run expr (arvados_cluster_ids) string (inputs.arvados_cluster_ids[0])
+
+ scatter arvbox_containers as container,
+ arvados_cluster_ids as cluster_id,
+ arvados_api_hosts as host
+ do run workflow(logincluster)
+ {
+ requirements {
+ EnvVarRequirement {
+ envDef: {
+ ARVBOX_CONTAINER: "$(inputs.container)"
+ }
+ }
+ }
+
+ run tool(container, cluster_id, logincluster, set_login = File("set_login.py")) {
+sh <<<
+set -x
+docker cp $(inputs.container):/var/lib/arvados/cluster_config.yml.override .
+chmod +w cluster_config.yml.override
+python $(inputs.set_login.path) cluster_config.yml.override $(inputs.cluster_id) $(inputs.logincluster)
+docker cp cluster_config.yml.override $(inputs.container):/var/lib/arvados
+>>>
+ return container as c
+ }
+ run tool(container, host, c) {
+sh <<<
+set -x
+arvbox hotreset
+while ! curl --fail --insecure --silent https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest >/dev/null ; do sleep 3 ; done
+export ARVADOS_API_HOST=$(inputs.host)
+export ARVADOS_API_TOKEN=\$(arvbox cat /var/lib/arvados/superuser_token)
+export ARVADOS_API_HOST_INSECURE=1
+ARVADOS_VIRTUAL_MACHINE_UUID=\$(arvbox cat /var/lib/arvados/vm-uuid)
+while ! python -c "import arvados ; arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()" 2>/dev/null ; do sleep 3; done
+>>>
+ return c as d
+ }
+ supertok = superuser_tok(container, d)
+ return supertok
+ }
+
+ report = run_test(arvados_api_hosts, superuser_tokens=supertok, fed_migrate)
+ return supertok, report
+}
\ No newline at end of file
diff --git a/sdk/python/tests/fed-migrate/run-test.cwl b/sdk/python/tests/fed-migrate/run-test.cwl
new file mode 100644
index 000000000..ea412ac8e
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/run-test.cwl
@@ -0,0 +1,127 @@
+#!/usr/bin/env cwl-runner
+class: Workflow
+cwlVersion: v1.0
+id: '#main'
+inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - default: arv-federation-migrate
+ id: fed_migrate
+ type: string
+outputs:
+ - id: out
+ outputSource: main_2/out
+ type: File
+requirements:
+ InlineJavascriptRequirement: {}
+ MultipleInputFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+ SubworkflowFeatureRequirement: {}
+steps:
+ - id: main_1
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ create_users:
+ default:
+ class: File
+ location: create_users.py
+ superuser_tokens:
+ source: superuser_tokens
+ out:
+ - ar
+ run:
+ arguments:
+ - python
+ - $(inputs.create_users)
+ - _script
+ class: CommandLineTool
+ id: main_1_embed
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: create_users
+ type: File
+ outputs:
+ - id: ar
+ outputBinding:
+ outputEval: $(inputs.arvados_api_hosts)
+ type:
+ items: string
+ type: array
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: |
+ {
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+ }
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: main_2
+ in:
+ ar:
+ source: main_1/ar
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ fed_migrate:
+ source: fed_migrate
+ host:
+ valueFrom: '$(inputs.arvados_api_hosts[0])'
+ superuser_tokens:
+ source: superuser_tokens
+ token:
+ valueFrom: '$(inputs.superuser_tokens[0])'
+ out:
+ - out
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--report'
+ - out
+ class: CommandLineTool
+ id: main_2_embed
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: ar
+ type:
+ items: string
+ type: array
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: out
+ outputBinding:
+ glob: out
+ type: File
+ requirements:
+ - class: EnvVarRequirement
+ envDef:
+ ARVADOS_API_HOST: $(inputs.host)
+ ARVADOS_API_TOKEN: $(inputs.token)
+
diff --git a/sdk/python/tests/fed-migrate/run-test.cwlex b/sdk/python/tests/fed-migrate/run-test.cwlex
new file mode 100644
index 000000000..3dda1fe7f
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/run-test.cwlex
@@ -0,0 +1,34 @@
+def workflow main(
+ arvados_api_hosts string[],
+ superuser_tokens string[],
+ fed_migrate="arv-federation-migrate"
+) {
+
+ run tool(arvados_api_hosts, superuser_tokens, create_users=File("create_users.py")) {
+ python $(inputs.create_users) <<<
+{
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+}
+>>>
+ return arvados_api_hosts as ar
+ }
+
+ run tool(arvados_api_hosts, superuser_tokens, ar,
+ fed_migrate,
+ host=$(inputs.arvados_api_hosts[0]),
+ token=$(inputs.superuser_tokens[0])) {
+ requirements {
+ EnvVarRequirement {
+ envDef: {
+ ARVADOS_API_HOST: "$(inputs.host)",
+ ARVADOS_API_TOKEN: "$(inputs.token)"
+ }
+ }
+ }
+ $(inputs.fed_migrate) --report report.csv
+ return File("report.csv") as report
+ }
+
+ return report
+}
\ No newline at end of file
diff --git a/sdk/python/tests/fed-migrate/set_login.py b/sdk/python/tests/fed-migrate/set_login.py
new file mode 100644
index 000000000..2900af182
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/set_login.py
@@ -0,0 +1,10 @@
+import json
+import sys
+
+f = open(sys.argv[1], "r+")
+j = json.load(f)
+j["Clusters"][sys.argv[2]]["Login"] = {"LoginCluster": sys.argv[3]}
+for r in j["Clusters"][sys.argv[2]]["RemoteClusters"]:
+ j["Clusters"][sys.argv[2]]["RemoteClusters"][r]["Insecure"] = True
+f.seek(0)
+json.dump(j, f)
diff --git a/sdk/python/tests/fed-migrate/superuser-tok.cwl b/sdk/python/tests/fed-migrate/superuser-tok.cwl
new file mode 100755
index 000000000..d2ce253a9
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/superuser-tok.cwl
@@ -0,0 +1,19 @@
+#!/usr/bin/env cwltool
+cwlVersion: v1.0
+class: CommandLineTool
+stdout: superuser_token.txt
+inputs:
+ container: string
+outputs:
+ superuser_token:
+ type: string
+ outputBinding:
+ glob: superuser_token.txt
+ loadContents: true
+ outputEval: $(self[0].contents.trim())
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: "$(inputs.container)"
+ InlineJavascriptRequirement: {}
+arguments: [arvbox, cat, /var/lib/arvados/superuser_token]
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index e309d999d..59fb3fc09 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -301,7 +301,6 @@ class User < ArvadosModel
user_updates = [
[AuthorizedKey, :owner_uuid],
[AuthorizedKey, :authorized_user_uuid],
- [Repository, :owner_uuid],
[Link, :owner_uuid],
[Link, :tail_uuid],
[Link, :head_uuid],
@@ -313,7 +312,6 @@ class User < ArvadosModel
AuthorizedKey.where(owner_uuid: uuid).destroy_all
AuthorizedKey.where(authorized_user_uuid: uuid).destroy_all
user_updates = [
- [Repository, :owner_uuid],
[Link, :owner_uuid],
[Link, :tail_uuid],
[Link, :head_uuid],
@@ -327,6 +325,20 @@ class User < ArvadosModel
klass.where(column => uuid).update_all(column => new_user.uuid)
end
+ # Need to update repository names to new username
+ old_repo_name_re = /^#{Regexp.escape(username)}\//
+ Repository.where(:owner_uuid => uuid).each do |repo|
+ repo.owner_uuid = new_user.uuid
+ repo_name_sub = "#{new_user.username}/"
+ name = repo.name.sub(old_repo_name_re, repo_name_sub)
+ while (conflict = Repository.where(:name => name).first) != nil
+ repo_name_sub += "migrated"
+ name = repo.name.sub(old_repo_name_re, repo_name_sub)
+ end
+ repo.name = name
+ repo.save!
+ end
+
# References to the merged user's "home project" are updated to
# point to new_owner_uuid.
ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
diff --git a/services/api/test/integration/users_test.rb b/services/api/test/integration/users_test.rb
index 6b7415407..11ebb3f4f 100644
--- a/services/api/test/integration/users_test.rb
+++ b/services/api/test/integration/users_test.rb
@@ -268,6 +268,7 @@ class UsersTest < ActionDispatch::IntegrationTest
headers: auth(:active))
assert_response(:success)
assert_equal(users(:project_viewer).uuid, json_response['owner_uuid'])
+ assert_equal("#{users(:project_viewer).username}/foo", json_response['name'])
get('/arvados/v1/groups/' + groups(:aproject).uuid,
params: {},
@@ -303,4 +304,39 @@ class UsersTest < ActionDispatch::IntegrationTest
assert_equal 'barney', json_response['username']
end
+ test 'merge with repository name conflict' do
+ post('/arvados/v1/groups',
+ params: {
+ group: {
+ group_class: 'project',
+ name: "active user's stuff",
+ },
+ },
+ headers: auth(:project_viewer))
+ assert_response(:success)
+ project_uuid = json_response['uuid']
+
+ post('/arvados/v1/repositories/',
+ params: { :repository => { :name => "#{users(:project_viewer).username}/foo", :owner_uuid => users(:project_viewer).uuid } },
+ headers: auth(:project_viewer))
+ assert_response(:success)
+
+ post('/arvados/v1/users/merge',
+ params: {
+ new_user_token: api_client_authorizations(:project_viewer_trustedclient).api_token,
+ new_owner_uuid: project_uuid,
+ redirect_to_new_user: true,
+ },
+ headers: auth(:active_trustedclient))
+ assert_response(:success)
+
+ get('/arvados/v1/repositories/' + repositories(:foo).uuid,
+ params: {},
+ headers: auth(:active))
+ assert_response(:success)
+ assert_equal(users(:project_viewer).uuid, json_response['owner_uuid'])
+ assert_equal("#{users(:project_viewer).username}/migratedfoo", json_response['name'])
+
+ end
+
end
commit f1051a2d445c680caade0321163dac88f084c130
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Sep 17 13:28:41 2019 -0400
15531: Fix remote token validate to use RemoteHosts.*.Insecure
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index 55db16a4b..e84a3d218 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -92,9 +92,11 @@ class ApiClientAuthorization < ArvadosModel
uuid_prefix+".arvadosapi.com")
end
- def self.make_http_client
+ def self.make_http_client(uuid_prefix:)
clnt = HTTPClient.new
- if Rails.configuration.TLS.Insecure
+
+ if uuid_prefix && (Rails.configuration.RemoteClusters[uuid_prefix].andand.Insecure ||
+ Rails.configuration.RemoteClusters['*'].andand.Insecure)
clnt.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
else
# Use system CA certificates
@@ -167,7 +169,7 @@ class ApiClientAuthorization < ArvadosModel
# by a remote cluster when the token absent or expired in our
# database. To begin, we need to ask the cluster that issued
# the token to [re]validate it.
- clnt = ApiClientAuthorization.make_http_client
+ clnt = ApiClientAuthorization.make_http_client(uuid_prefix: token_uuid_prefix)
host = remote_host(uuid_prefix: token_uuid_prefix)
if !host
commit 5a280d55dae6daaba3679a55e33d07561ff1c016
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Sep 13 16:39:59 2019 -0400
15531: Working on properly migrating usernames
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/federation_migrate.py b/sdk/python/arvados/commands/federation_migrate.py
index 9211f0608..386c0ef9b 100755
--- a/sdk/python/arvados/commands/federation_migrate.py
+++ b/sdk/python/arvados/commands/federation_migrate.py
@@ -160,11 +160,24 @@ def main():
if len(userhome) == 5 and userhome not in clusters:
print("(%s) Cannot migrate %s, unknown home cluster %s (typo?)" % (email, old_user_uuid, userhome))
continue
- print("(%s) No user listed with same email to migrate %s to %s, will create new user" % (email, old_user_uuid, userhome))
+ print("(%s) No user listed with same email to migrate %s to %s, will create new user with username '%s'" % (email, old_user_uuid, userhome, username))
if not args.dry_run:
newhomecluster = userhome[0:5]
homearv = clusters[userhome]
- user = homearv.users().create({"email": email, "username": username}).execute()
+ user = None
+ try:
+ user = homearv.users().create(body={"user": {"email": email, "username": username}}).execute()
+ except arvados.errors.ApiError as e:
+ if "Username" in str(e):
+ other = homearv.users().list(filters=[["username", "=", username]]).execute()
+ if other['items'] and other['items'][0]['email'] == email:
+ conflicting_user = other['items'][0]
+ homearv.users().update(uuid=conflicting_user["uuid"], body={"user": {"username": username+"migrate"}}).execute()
+ user = homearv.users().create(body={"user": {"email": email, "username": username}}).execute()
+ if not user:
+ print("(%s) Could not create user: %s" % (email, str(e)))
+ continue
+
candidates.append((email, username, user["uuid"], userhome))
else:
candidates.append((email, username, "%s-tpzed-xfakexfakexfake" % (userhome[0:5]), userhome))
@@ -229,7 +242,6 @@ def main():
print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
continue
-
print("(%s) Migrating %s to %s on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
try:
@@ -242,9 +254,16 @@ def main():
migratearv.users().merge(old_user_uuid=old_user_uuid,
new_user_uuid=new_user_uuid,
new_owner_uuid=grp["uuid"],
- redirect_to_new_user=True).execute()
+ redirect_to_new_user=old_user_uuid.startswith(migratecluster)).execute()
except arvados.errors.ApiError as e:
print("(%s) Error migrating user: %s" % (email, e))
+ if newuser['username'] != username:
+ print("%s != %s" % (newuser['username'], username))
+ try:
+ migratearv.users().update(uuid=new_user_uuid, body={"user": {"username": username}}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, new_user_uuid, username, migratecluster, e))
+
if __name__ == "__main__":
main()
diff --git a/services/api/app/models/repository.rb b/services/api/app/models/repository.rb
index e6a079540..46f2de6ee 100644
--- a/services/api/app/models/repository.rb
+++ b/services/api/app/models/repository.rb
@@ -92,7 +92,7 @@ class Repository < ArvadosModel
end
if not (/^#{prefix_match}[A-Za-z][A-Za-z0-9]*$/.match(name))
errors.add(:name,
- "#{errmsg_start} a letter followed by alphanumerics")
+ "#{errmsg_start} a letter followed by alphanumerics, expected pattern '#{prefix_match}[A-Za-z][A-Za-z0-9]*' but was '#{name}'")
false
end
end
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index 65cb75306..e309d999d 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -339,7 +339,7 @@ class User < ArvadosModel
klass.where(owner_uuid: uuid).update_all(owner_uuid: new_owner_uuid)
end
- update_attributes!(redirect_to_user_uuid: new_user.uuid)
+ update_attributes!(redirect_to_user_uuid: new_user.uuid, username: nil)
invalidate_permissions_cache
end
end
commit 86ca72db7b721b14b4cc658755d012dc115a5988
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Sep 13 14:09:04 2019 -0400
15531: Implement merge with redirect_to_new_user=false
When false, delete credentials of old user instead of migrating them.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index 4a345f363..0ef129481 100644
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -181,12 +181,10 @@ class Arvados::V1::UsersController < ApplicationController
return send_error("cannot merge remote user to other with redirect_to_new_user=true", status: 422)
end
- if !redirect
- return send_error("merge with redirect_to_new_user=false is not yet supported", status: 422)
- end
-
act_as_system_user do
- @object.merge(new_owner_uuid: params[:new_owner_uuid], redirect_to_user_uuid: redirect && new_user.uuid)
+ @object.merge(new_owner_uuid: params[:new_owner_uuid],
+ redirect_to_user_uuid: new_user.uuid,
+ redirect_auth: redirect)
end
show
end
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index 4493f038c..65cb75306 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -280,7 +280,7 @@ class User < ArvadosModel
#
# current_user must have admin privileges, i.e., the caller is
# responsible for checking permission to do this.
- def merge(new_owner_uuid:, redirect_to_user_uuid:)
+ def merge(new_owner_uuid:, redirect_to_user_uuid:, redirect_auth:)
raise PermissionDeniedError if !current_user.andand.is_admin
raise "not implemented" if !redirect_to_user_uuid
transaction(requires_new: true) do
@@ -291,23 +291,39 @@ class User < ArvadosModel
raise "user does not exist" if !new_user
raise "cannot merge to an already merged user" if new_user.redirect_to_user_uuid
- # Existing API tokens are updated to authenticate to the new
- # user.
- ApiClientAuthorization.
- where(user_id: id).
- update_all(user_id: new_user.id)
+ if redirect_auth
+ # Existing API tokens and ssh keys are updated to authenticate
+ # to the new user.
+ ApiClientAuthorization.
+ where(user_id: id).
+ update_all(user_id: new_user.id)
+
+ user_updates = [
+ [AuthorizedKey, :owner_uuid],
+ [AuthorizedKey, :authorized_user_uuid],
+ [Repository, :owner_uuid],
+ [Link, :owner_uuid],
+ [Link, :tail_uuid],
+ [Link, :head_uuid],
+ ]
+ else
+ # Destroy API tokens and ssh keys associated with the old
+ # user.
+ ApiClientAuthorization.where(user_id: id).destroy_all
+ AuthorizedKey.where(owner_uuid: uuid).destroy_all
+ AuthorizedKey.where(authorized_user_uuid: uuid).destroy_all
+ user_updates = [
+ [Repository, :owner_uuid],
+ [Link, :owner_uuid],
+ [Link, :tail_uuid],
+ [Link, :head_uuid],
+ ]
+ end
# References to the old user UUID in the context of a user ID
# (rather than a "home project" in the project hierarchy) are
# updated to point to the new user.
- [
- [AuthorizedKey, :owner_uuid],
- [AuthorizedKey, :authorized_user_uuid],
- [Repository, :owner_uuid],
- [Link, :owner_uuid],
- [Link, :tail_uuid],
- [Link, :head_uuid],
- ].each do |klass, column|
+ user_updates.each do |klass, column|
klass.where(column => uuid).update_all(column => new_user.uuid)
end
diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb
index 60696b98a..4e19988de 100644
--- a/services/api/test/functional/arvados/v1/users_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/users_controller_test.rb
@@ -817,14 +817,21 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
end
end
- test "refuse to merge with redirect_to_user_uuid=false (not yet supported)" do
+ test "merge with redirect_to_user_uuid=false" do
authorize_with :project_viewer_trustedclient
+ tok = api_client_authorizations(:project_viewer).api_token
post :merge, params: {
new_user_token: api_client_authorizations(:active_trustedclient).api_token,
new_owner_uuid: users(:active).uuid,
redirect_to_new_user: false,
}
- assert_response(422)
+ assert_response(:success)
+ assert_equal(users(:active).uuid, User.unscoped.find_by_uuid(users(:project_viewer).uuid).redirect_to_user_uuid)
+
+ # because redirect_to_new_user=false, token owned by
+ # project_viewer should be deleted
+ auth = ApiClientAuthorization.validate(token: tok)
+ assert_nil(auth)
end
test "refuse to merge user into self" do
commit 3b9310734cf6b29b35caff5dfbe64d88bc4789bf
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Sep 11 11:29:18 2019 -0400
15531: Federation migrate script wip
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/federation_migrate.py b/sdk/python/arvados/commands/federation_migrate.py
index 1daf6beb7..9211f0608 100755
--- a/sdk/python/arvados/commands/federation_migrate.py
+++ b/sdk/python/arvados/commands/federation_migrate.py
@@ -11,51 +11,67 @@ import sys
import argparse
import hmac
import urllib.parse
+import os
def main():
parser = argparse.ArgumentParser(description='Migrate users to federated identity, see https://doc.arvados.org/admin/merge-remote-account.html')
- parser.add_argument('--tokens', type=str, required=True)
+ parser.add_argument('--tokens', type=str, required=False)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--report', type=str, help="Generate report .csv file listing users by email address and their associated Arvados accounts")
group.add_argument('--migrate', type=str, help="Consume report .csv and migrate users to designated Arvados accounts")
+ group.add_argument('--dry-run', type=str, help="Consume report .csv and report how user would be migrated to designated Arvados accounts")
group.add_argument('--check', action="store_true", help="Check that tokens are usable and the federation is well connected")
args = parser.parse_args()
clusters = {}
errors = []
- print("Reading %s" % args.tokens)
- with open(args.tokens, "rt") as f:
- for r in csv.reader(f):
- host = r[0]
- token = r[1]
- print("Contacting %s" % (host))
- arv = arvados.api(host=host, token=token, cache=False)
- try:
- cur = arv.users().current().execute()
- arv.api_client_authorizations().list(limit=1).execute()
- except arvados.errors.ApiError as e:
- errors.append("checking token for %s: %s" % (host, e))
- errors.append(' This script requires a token issued to a trusted client in order to manipulate access tokens.')
- errors.append(' See "Trusted client setting" in https://doc.arvados.org/install/install-workbench-app.html')
- errors.append(' and https://doc.arvados.org/api/tokens.html')
- continue
-
- if not cur["is_admin"]:
- errors.append("Not admin of %s" % host)
- continue
-
- clusters[arv._rootDesc["uuidPrefix"]] = arv
-
+ loginCluster = None
+ if args.tokens:
+ print("Reading %s" % args.tokens)
+ with open(args.tokens, "rt") as f:
+ for r in csv.reader(f):
+ host = r[0]
+ token = r[1]
+ print("Contacting %s" % (host))
+ arv = arvados.api(host=host, token=token, cache=False)
+ clusters[arv._rootDesc["uuidPrefix"]] = arv
+ else:
+ arv = arvados.api(cache=False)
+ rh = arv._rootDesc["remoteHosts"]
+ for k,v in rh.items():
+ arv = arvados.api(host=v, token=os.environ["ARVADOS_API_TOKEN"], cache=False)
+ config = arv.configs().get().execute()
+ if config["Login"]["LoginCluster"] != "" and loginCluster is None:
+ loginCluster = config["Login"]["LoginCluster"]
+ clusters[k] = arv
print("Checking that the federation is well connected")
- for v in clusters.values():
+ for arv in clusters.values():
+ config = arv.configs().get().execute()
+ if loginCluster and config["Login"]["LoginCluster"] != loginCluster and config["ClusterID"] != loginCluster:
+ errors.append("Inconsistent login cluster configuration, expected '%s' on %s but was '%s'" % (loginCluster, config["ClusterID"], config["Login"]["LoginCluster"]))
+ continue
+ try:
+ cur = arv.users().current().execute()
+ #arv.api_client_authorizations().list(limit=1).execute()
+ except arvados.errors.ApiError as e:
+ errors.append("checking token for %s %s" % (arv._rootDesc["rootUrl"], e))
+ errors.append(' This script requires a token issued to a trusted client in order to manipulate access tokens.')
+ errors.append(' See "Trusted client setting" in https://doc.arvados.org/install/install-workbench-app.html')
+ errors.append(' and https://doc.arvados.org/api/tokens.html')
+ continue
+
+ if not cur["is_admin"]:
+ errors.append("Not admin of %s" % host)
+ continue
+
for r in clusters:
- if r != v._rootDesc["uuidPrefix"] and r not in v._rootDesc["remoteHosts"]:
- errors.append("%s is missing from remoteHosts of %s" % (r, v._rootDesc["uuidPrefix"]))
- for r in v._rootDesc["remoteHosts"]:
+ if r != arv._rootDesc["uuidPrefix"] and r not in arv._rootDesc["remoteHosts"]:
+ errors.append("%s is missing from remoteHosts of %s" % (r, arv._rootDesc["uuidPrefix"]))
+ for r in arv._rootDesc["remoteHosts"]:
if r != "*" and r not in clusters:
- print("WARNING: %s is federated with %s but %s is missing from the tokens file or the token is invalid" % (v._rootDesc["uuidPrefix"], r, r))
+ print("WARNING: %s is federated with %s but %s is missing from the tokens file or the token is invalid" % (arv._rootDesc["uuidPrefix"], r, r))
if errors:
for e in errors:
@@ -77,9 +93,9 @@ def main():
out = csv.writer(open(args.report, "wt"))
- out.writerow(("email", "user uuid", "primary cluster/user"))
+ out.writerow(("email", "username", "user uuid", "home cluster"))
- users = sorted(users, key=lambda u: u["email"]+"::"+u["uuid"])
+ users = sorted(users, key=lambda u: u["email"]+"::"+(u["username"] or "")+"::"+u["uuid"])
accum = []
lastemail = None
@@ -98,7 +114,7 @@ def main():
if a["uuid"] != homeuuid:
homeuuid = ""
for a in accum:
- out.writerow((a["email"], a["uuid"], homeuuid[0:5]))
+ out.writerow((a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5]))
lastemail = u["email"]
accum = [u]
@@ -109,14 +125,17 @@ def main():
if a["uuid"] != homeuuid:
homeuuid = ""
for a in accum:
- out.writerow((a["email"], a["uuid"], homeuuid[0:5]))
+ out.writerow((a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5]))
print("Wrote %s" % args.report)
- if args.migrate:
+ if args.migrate or args.dry_run:
+ if args.dry_run:
+ print("Performing dry run")
+
rows = []
by_email = {}
- with open(args.migrate, "rt") as f:
+ with open(args.migrate or args.dry_run, "rt") as f:
for r in csv.reader(f):
if r[0] == "email":
continue
@@ -125,8 +144,9 @@ def main():
rows.append(r)
for r in rows:
email = r[0]
- old_user_uuid = r[1]
- userhome = r[2]
+ username = r[1]
+ old_user_uuid = r[2]
+ userhome = r[3]
if userhome == "":
print("(%s) Skipping %s, no home cluster specified" % (email, old_user_uuid))
@@ -134,80 +154,97 @@ def main():
continue
candidates = []
for b in by_email[email]:
- if b[1].startswith(userhome):
+ if b[2].startswith(userhome):
candidates.append(b)
if len(candidates) == 0:
if len(userhome) == 5 and userhome not in clusters:
print("(%s) Cannot migrate %s, unknown home cluster %s (typo?)" % (email, old_user_uuid, userhome))
+ continue
+ print("(%s) No user listed with same email to migrate %s to %s, will create new user" % (email, old_user_uuid, userhome))
+ if not args.dry_run:
+ newhomecluster = userhome[0:5]
+ homearv = clusters[userhome]
+ user = homearv.users().create({"email": email, "username": username}).execute()
+ candidates.append((email, username, user["uuid"], userhome))
else:
- print("(%s) No user listed with same email to migrate %s to %s" % (email, old_user_uuid, userhome))
- continue
+ candidates.append((email, username, "%s-tpzed-xfakexfakexfake" % (userhome[0:5]), userhome))
if len(candidates) > 1:
print("(%s) Multiple users listed to migrate %s to %s, use full uuid" % (email, old_user_uuid, userhome))
continue
- new_user_uuid = candidates[0][1]
+ new_user_uuid = candidates[0][2]
# cluster where the migration is happening
- migratecluster = old_user_uuid[0:5]
- migratearv = clusters[migratecluster]
-
- # the user's new home cluster
- newhomecluster = userhome[0:5]
- homearv = clusters[newhomecluster]
-
- # create a token for the new user and salt it for the
- # migration cluster, then use it to access the migration
- # cluster as the new user once before merging to ensure
- # the new user is known on that cluster.
- try:
- newtok = homearv.api_client_authorizations().create(body={
- "api_client_authorization": {'owner_uuid': new_user_uuid}}).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Could not create API token for %s: %s" % (email, new_user_uuid, e))
- continue
-
- salted = 'v2/' + newtok["uuid"] + '/' + hmac.new(newtok["api_token"].encode(),
- msg=migratecluster.encode(),
- digestmod='sha1').hexdigest()
- try:
- ru = urllib.parse.urlparse(migratearv._rootDesc["rootUrl"])
- newuser = arvados.api(host=ru.netloc, token=salted).users().current().execute()
- except arvados.errors.ApiError as e:
- print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
- continue
+ for arv in clusters.values():
+ migratecluster = arv._rootDesc["uuidPrefix"]
+ migratearv = clusters[migratecluster]
+
+ # the user's new home cluster
+ newhomecluster = userhome[0:5]
+ homearv = clusters[newhomecluster]
+
+ # create a token for the new user and salt it for the
+ # migration cluster, then use it to access the migration
+ # cluster as the new user once before merging to ensure
+ # the new user is known on that cluster.
+ try:
+ if not args.dry_run:
+ newtok = homearv.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': new_user_uuid}}).execute()
+ else:
+ newtok = {"uuid": "dry-run", "api_token": "12345"}
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not create API token for %s: %s" % (email, new_user_uuid, e))
+ continue
- try:
- olduser = migratearv.users().get(uuid=old_user_uuid).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Could not retrieve user %s from %s, user may have already been migrated: %s" % (email, old_user_uuid, migratecluster, e))
- continue
+ salted = 'v2/' + newtok["uuid"] + '/' + hmac.new(newtok["api_token"].encode(),
+ msg=migratecluster.encode(),
+ digestmod='sha1').hexdigest()
+ try:
+ ru = urllib.parse.urlparse(migratearv._rootDesc["rootUrl"])
+ if not args.dry_run:
+ newuser = arvados.api(host=ru.netloc, token=salted).users().current().execute()
+ else:
+ newuser = {"is_active": True}
+ except arvados.errors.ApiError as e:
+ print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
+ continue
- if not newuser["is_active"]:
- print("(%s) Activating user %s on %s" % (email, new_user_uuid, migratecluster))
try:
- migratearv.users().update(uuid=new_user_uuid, body={"is_active": True}).execute()
+ olduser = migratearv.users().get(uuid=old_user_uuid).execute()
except arvados.errors.ApiError as e:
- print("(%s) Could not activate user %s on %s: %s" % (email, new_user_uuid, migratecluster, e))
+ if e.resp.status != 404:
+ print("(%s) Could not retrieve user %s from %s, user may have already been migrated: %s" % (email, old_user_uuid, migratecluster, e))
+ continue
+
+ if not newuser["is_active"]:
+ print("(%s) Activating user %s on %s" % (email, new_user_uuid, migratecluster))
+ try:
+ if not args.dry_run:
+ migratearv.users().update(uuid=new_user_uuid, body={"is_active": True}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not activate user %s on %s: %s" % (email, new_user_uuid, migratecluster, e))
+ continue
+
+ if olduser["is_admin"] and not newuser["is_admin"]:
+ print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
continue
- if olduser["is_admin"] and not newuser["is_admin"]:
- print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
- continue
- print("(%s) Migrating %s to %s on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
-
- try:
- grp = migratearv.groups().create(body={
- "owner_uuid": new_user_uuid,
- "name": "Migrated from %s (%s)" % (email, old_user_uuid),
- "group_class": "project"
- }, ensure_unique_name=True).execute()
- migratearv.users().merge(old_user_uuid=old_user_uuid,
- new_user_uuid=new_user_uuid,
- new_owner_uuid=grp["uuid"],
- redirect_to_new_user=True).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Error migrating user: %s" % (email, e))
+ print("(%s) Migrating %s to %s on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
+
+ try:
+ if not args.dry_run:
+ grp = migratearv.groups().create(body={
+ "owner_uuid": new_user_uuid,
+ "name": "Migrated from %s (%s)" % (email, old_user_uuid),
+ "group_class": "project"
+ }, ensure_unique_name=True).execute()
+ migratearv.users().merge(old_user_uuid=old_user_uuid,
+ new_user_uuid=new_user_uuid,
+ new_owner_uuid=grp["uuid"],
+ redirect_to_new_user=True).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error migrating user: %s" % (email, e))
if __name__ == "__main__":
main()
commit 2fa07301c16fb0c3efe3812e9ad8e058f257a3ea
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Sep 10 14:22:15 2019 -0400
15531: Add exported config to discovery document
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 14abfae03..84a674607 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -401,6 +401,28 @@ class Arvados::V1::SchemaController < ApplicationController
end
end
end
+
+ discovery[:resources]['configs'] = {
+ methods: {
+ get: {
+ id: "arvados.configs.get",
+ path: "config",
+ httpMethod: "GET",
+ description: "Get public config",
+ parameters: {
+ },
+ parameterOrder: [
+ ],
+ response: {
+ },
+ scopes: [
+ "https://api.curoverse.com/auth/arvados",
+ "https://api.curoverse.com/auth/arvados.readonly"
+ ]
+ },
+ }
+ }
+
Rails.configuration.API.DisabledAPIs.each do |method, _|
ctrl, action = method.to_s.split('.', 2)
discovery[:resources][ctrl][:methods].delete(action.to_sym)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list