[ARVADOS] updated: 1.3.0-856-g36a33fe21
Git user
git at public.curoverse.com
Wed May 15 18:57:31 UTC 2019
Summary of changes:
sdk/python/arvados/commands/federation_migrate.py | 137 ++++++++++++++++------
1 file changed, 100 insertions(+), 37 deletions(-)
via 36a33fe214b13cbe9388df92fc7bf83c81f42d11 (commit)
from ec96bfb7ddad8d27862ace0c382f9bd33679f96c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 36a33fe214b13cbe9388df92fc7bf83c81f42d11
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed May 15 14:38:15 2019 -0400
15061: Code cleanup, improve validity checking & error handling
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/federation_migrate.py b/sdk/python/arvados/commands/federation_migrate.py
index af29f3a64..1daf6beb7 100755
--- a/sdk/python/arvados/commands/federation_migrate.py
+++ b/sdk/python/arvados/commands/federation_migrate.py
@@ -5,10 +5,12 @@
import arvados
import arvados.util
+import arvados.errors
import csv
import sys
import argparse
import hmac
+import urllib.parse
def main():
@@ -21,34 +23,47 @@ def main():
args = parser.parse_args()
clusters = {}
-
+ errors = []
print("Reading %s" % args.tokens)
with open(args.tokens, "rt") as f:
for r in csv.reader(f):
host = r[0]
token = r[1]
print("Contacting %s" % (host))
- arv = arvados.api(host=host, token=token)
- clusters[arv._rootDesc["uuidPrefix"]] = arv
- cur = arv.users().current().execute()
+ arv = arvados.api(host=host, token=token, cache=False)
+ try:
+ cur = arv.users().current().execute()
+ arv.api_client_authorizations().list(limit=1).execute()
+ except arvados.errors.ApiError as e:
+ errors.append("checking token for %s: %s" % (host, e))
+ errors.append(' This script requires a token issued to a trusted client in order to manipulate access tokens.')
+ errors.append(' See "Trusted client setting" in https://doc.arvados.org/install/install-workbench-app.html')
+ errors.append(' and https://doc.arvados.org/api/tokens.html')
+ continue
+
if not cur["is_admin"]:
- raise Exception("Not admin of %s" % host)
+ errors.append("Not admin of %s" % host)
+ continue
+
+ clusters[arv._rootDesc["uuidPrefix"]] = arv
+
print("Checking that the federation is well connected")
- fail = False
for v in clusters.values():
for r in clusters:
if r != v._rootDesc["uuidPrefix"] and r not in v._rootDesc["remoteHosts"]:
- print("ERROR: %s is missing from remoteHosts of %s" % (r, v._rootDesc["uuidPrefix"]))
- fail = True
+ errors.append("%s is missing from remoteHosts of %s" % (r, v._rootDesc["uuidPrefix"]))
for r in v._rootDesc["remoteHosts"]:
if r != "*" and r not in clusters:
- print("WARNING: %s is federated with %s but %s is not in the tokens file" % (v._rootDesc["uuidPrefix"], r, r))
+ print("WARNING: %s is federated with %s but %s is missing from the tokens file or the token is invalid" % (v._rootDesc["uuidPrefix"], r, r))
- if fail:
+ if errors:
+ for e in errors:
+ print("ERROR: "+str(e))
exit(1)
if args.check:
+ print("Tokens file passed checks")
exit(0)
if args.report:
@@ -109,42 +124,90 @@ def main():
by_email[r[0]].append(r)
rows.append(r)
for r in rows:
- if r[2] == "":
- print("(%s) Skipping %s, no home cluster specified" % (r[0], r[1]))
- if r[1].startswith(r[2]):
+ email = r[0]
+ old_user_uuid = r[1]
+ userhome = r[2]
+
+ if userhome == "":
+ print("(%s) Skipping %s, no home cluster specified" % (email, old_user_uuid))
+ if old_user_uuid.startswith(userhome):
continue
candidates = []
- for b in by_email[r[0]]:
- if b[1].startswith(r[2]):
+ for b in by_email[email]:
+ if b[1].startswith(userhome):
candidates.append(b)
if len(candidates) == 0:
- print("(%s) No user listed to migrate %s to %s" % (r[0], r[1], r[2]))
+ if len(userhome) == 5 and userhome not in clusters:
+ print("(%s) Cannot migrate %s, unknown home cluster %s (typo?)" % (email, old_user_uuid, userhome))
+ else:
+ print("(%s) No user listed with same email to migrate %s to %s" % (email, old_user_uuid, userhome))
continue
if len(candidates) > 1:
- print("(%s) Multiple users listed to migrate %s to %s, use full uuid" % (r[0], r[1], r[2]))
+ print("(%s) Multiple users listed to migrate %s to %s, use full uuid" % (email, old_user_uuid, userhome))
continue
new_user_uuid = candidates[0][1]
- print("(%s) Migrating %s to %s" % (r[0], r[1], new_user_uuid))
- oldcluster = r[1][0:5]
- newhomecluster = r[2][0:5]
+
+ # cluster where the migration is happening
+ migratecluster = old_user_uuid[0:5]
+ migratearv = clusters[migratecluster]
+
+ # the user's new home cluster
+ newhomecluster = userhome[0:5]
homearv = clusters[newhomecluster]
- # create a token
- newtok = homearv.api_client_authorizations().create(body={"api_client_authorization": {'owner_uuid': new_user_uuid}}).execute()
- salted = 'v2/' + newtok["uuid"] + '/' + hmac.new(newtok["api_token"].encode(), msg=oldcluster.encode(), digestmod='sha1').hexdigest()
- arvados.api(host=arv._rootDesc["rootUrl"][8:-1], token=salted).users().current().execute()
-
- # now migrate from local user to remote user.
- arv = clusters[oldcluster]
-
- grp = arv.groups().create(body={
- "owner_uuid": new_user_uuid,
- "name": "Migrated from %s (%s)" % (r[0], r[1]),
- "group_class": "project"
- }, ensure_unique_name=True).execute()
- arv.users().merge(old_user_uuid=r[1],
- new_user_uuid=new_user_uuid,
- new_owner_uuid=grp["uuid"],
- redirect_to_new_user=True).execute()
+
+ # create a token for the new user and salt it for the
+ # migration cluster, then use it to access the migration
+ # cluster as the new user once before merging to ensure
+ # the new user is known on that cluster.
+ try:
+ newtok = homearv.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': new_user_uuid}}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not create API token for %s: %s" % (email, new_user_uuid, e))
+ continue
+
+ salted = 'v2/' + newtok["uuid"] + '/' + hmac.new(newtok["api_token"].encode(),
+ msg=migratecluster.encode(),
+ digestmod='sha1').hexdigest()
+ try:
+ ru = urllib.parse.urlparse(migratearv._rootDesc["rootUrl"])
+ newuser = arvados.api(host=ru.netloc, token=salted).users().current().execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
+ continue
+
+ try:
+ olduser = migratearv.users().get(uuid=old_user_uuid).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not retrieve user %s from %s, user may have already been migrated: %s" % (email, old_user_uuid, migratecluster, e))
+ continue
+
+ if not newuser["is_active"]:
+ print("(%s) Activating user %s on %s" % (email, new_user_uuid, migratecluster))
+ try:
+ migratearv.users().update(uuid=new_user_uuid, body={"is_active": True}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not activate user %s on %s: %s" % (email, new_user_uuid, migratecluster, e))
+ continue
+
+ if olduser["is_admin"] and not newuser["is_admin"]:
+ print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
+ continue
+
+ print("(%s) Migrating %s to %s on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
+
+ try:
+ grp = migratearv.groups().create(body={
+ "owner_uuid": new_user_uuid,
+ "name": "Migrated from %s (%s)" % (email, old_user_uuid),
+ "group_class": "project"
+ }, ensure_unique_name=True).execute()
+ migratearv.users().merge(old_user_uuid=old_user_uuid,
+ new_user_uuid=new_user_uuid,
+ new_owner_uuid=grp["uuid"],
+ redirect_to_new_user=True).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error migrating user: %s" % (email, e))
if __name__ == "__main__":
main()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list