[ARVADOS] updated: 1.3.0-1629-gde3276d1b
Git user
git at public.curoverse.com
Fri Sep 27 18:00:37 UTC 2019
Summary of changes:
sdk/python/arvados/commands/federation_migrate.py | 405 ++++++++++++----------
sdk/python/tests/fed-migrate/run-test.cwl | 291 ----------------
tools/arvbox/bin/arvbox | 1 +
3 files changed, 232 insertions(+), 465 deletions(-)
delete mode 100644 sdk/python/tests/fed-migrate/run-test.cwl
via de3276d1b2067be4aa40b51605c2ce509ea80915 (commit)
from f3c6c6e5853ad28cc2edfb70c60f88bd1547414d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit de3276d1b2067be4aa40b51605c2ce509ea80915
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Sep 27 13:59:56 2019 -0400
15531: Refactor federation-migrate
Split into functions for readability.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/python/arvados/commands/federation_migrate.py b/sdk/python/arvados/commands/federation_migrate.py
index 3b3a7ee66..a7d9414d9 100755
--- a/sdk/python/arvados/commands/federation_migrate.py
+++ b/sdk/python/arvados/commands/federation_migrate.py
@@ -12,18 +12,14 @@ import argparse
import hmac
import urllib.parse
import os
+from arvados._version import __version__
-def main():
-
- parser = argparse.ArgumentParser(description='Migrate users to federated identity, see https://doc.arvados.org/admin/merge-remote-account.html')
- parser.add_argument('--tokens', type=str, required=False)
- group = parser.add_mutually_exclusive_group(required=True)
- group.add_argument('--report', type=str, help="Generate report .csv file listing users by email address and their associated Arvados accounts")
- group.add_argument('--migrate', type=str, help="Consume report .csv and migrate users to designated Arvados accounts")
- group.add_argument('--dry-run', type=str, help="Consume report .csv and report how user would be migrated to designated Arvados accounts")
- group.add_argument('--check', action="store_true", help="Check that tokens are usable and the federation is well connected")
- args = parser.parse_args()
+EMAIL=0
+USERNAME=1
+UUID=2
+HOMECLUSTER=3
+def connect_clusters(args):
clusters = {}
errors = []
loginCluster = None
@@ -31,6 +27,8 @@ def main():
print("Reading %s" % args.tokens)
with open(args.tokens, "rt") as f:
for r in csv.reader(f):
+ if len(r) != 2:
+ continue
host = r[0]
token = r[1]
print("Contacting %s" % (host))
@@ -44,25 +42,28 @@ def main():
for k,v in rh.items():
arv = arvados.api(host=v, token=token, cache=False, insecure=os.environ.get("ARVADOS_API_HOST_INSECURE"))
- config = arv.configs().get().execute()
- if config["Login"]["LoginCluster"] != "" and loginCluster is None:
- loginCluster = config["Login"]["LoginCluster"]
clusters[k] = arv
+ for _, arv in clusters.items():
+ config = arv.configs().get().execute()
+ if config["Login"]["LoginCluster"] != "" and loginCluster is None:
+ loginCluster = config["Login"]["LoginCluster"]
+
print("Checking that the federation is well connected")
for arv in clusters.values():
config = arv.configs().get().execute()
if loginCluster and config["Login"]["LoginCluster"] != loginCluster and config["ClusterID"] != loginCluster:
errors.append("Inconsistent login cluster configuration, expected '%s' on %s but was '%s'" % (loginCluster, config["ClusterID"], config["Login"]["LoginCluster"]))
continue
+
+ if arv._rootDesc["revision"] < "20190926":
+ errors.append("Arvados API server revision on cluster '%s' is too old, must be updated to at least Arvados 1.5 before running migration." % config["ClusterID"])
+ continue
+
try:
cur = arv.users().current().execute()
- #arv.api_client_authorizations().list(limit=1).execute()
except arvados.errors.ApiError as e:
errors.append("checking token for %s %s" % (arv._rootDesc["rootUrl"], e))
- errors.append(' This script requires a token issued to a trusted client in order to manipulate access tokens.')
- errors.append(' See "Trusted client setting" in https://doc.arvados.org/install/install-workbench-app.html')
- errors.append(' and https://doc.arvados.org/api/tokens.html')
continue
if not cur["is_admin"]:
@@ -76,17 +77,13 @@ def main():
if r != "*" and r not in clusters:
print("WARNING: %s is federated with %s but %s is missing from the tokens file or the token is invalid" % (arv._rootDesc["uuidPrefix"], r, r))
- if errors:
- for e in errors:
- print("ERROR: "+str(e))
- exit(1)
+ return clusters, errors, loginCluster
- if args.check:
- print("Tokens file passed checks")
- exit(0)
+def fetch_users(clusters, loginCluster):
rows = []
by_email = {}
+ by_username = {}
users = []
for c, arv in clusters.items():
@@ -96,10 +93,34 @@ def main():
if l["uuid"].startswith(c):
users.append(l)
+ # Users list is sorted by email
+ # Go through users and collect users with same email
+ # when we see a different email (or get to the end)
+ # call add_accum_rows() to generate the report rows with
+ # the "home cluster" set, and also fill in the by_email table.
+
users = sorted(users, key=lambda u: u["email"]+"::"+(u["username"] or "")+"::"+u["uuid"])
accum = []
lastemail = None
+
+ def add_accum_rows():
+ homeuuid = None
+ for a in accum:
+ uuids = set(a["uuid"] for a in accum)
+ homeuuid = ((len(uuids) == 1) and uuids.pop()) or ""
+ for a in accum:
+ r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
+ by_email.setdefault(a["email"], {})
+ by_email[a["email"]][a["uuid"]] = r
+ homeuuid_and_username = "%s::%s" % (r[HOMECLUSTER], a["username"])
+ if homeuuid_and_username not in by_username:
+ by_username[homeuuid_and_username] = a["email"]
+ elif by_username[homeuuid_and_username] != a["email"]:
+ print("ERROR: the username '%s' is listed for both '%s' and '%s' on cluster '%s'" % (r[USERNAME], r[EMAIL], by_username[homeuuid_and_username], r[HOMECLUSTER]))
+ exit(1)
+ rows.append(r)
+
for u in users:
if u["uuid"].endswith("-anonymouspublic") or u["uuid"].endswith("-000000000000000"):
continue
@@ -108,31 +129,181 @@ def main():
if u["email"] == lastemail:
accum.append(u)
else:
- homeuuid = None
- for a in accum:
- if homeuuid is None:
- homeuuid = a["uuid"]
- if a["uuid"] != homeuuid:
- homeuuid = ""
- for a in accum:
- r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
- by_email.setdefault(a["email"], {})
- by_email[a["email"]][a["uuid"]] = r
- rows.append(r)
+ add_accum_rows()
lastemail = u["email"]
accum = [u]
- homeuuid = None
- for a in accum:
- if homeuuid is None:
- homeuuid = a["uuid"]
- if a["uuid"] != homeuuid:
- homeuuid = ""
- for a in accum:
- r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
- by_email.setdefault(a["email"], {})
- by_email[a["email"]][a["uuid"]] = r
- rows.append(r)
+ add_accum_rows()
+
+ return rows, by_email, by_username
+
+
+def read_migrations(args, by_email, by_username):
+ rows = []
+ with open(args.migrate or args.dry_run, "rt") as f:
+ for r in csv.reader(f):
+ if r[EMAIL] == "email":
+ continue
+ by_email.setdefault(r[EMAIL], {})
+ by_email[r[EMAIL]][r[UUID]] = r
+
+ homeuuid_and_username = "%s::%s" % (r[HOMECLUSTER], r[USERNAME])
+ if homeuuid_and_username not in by_username:
+ by_username[homeuuid_and_username] = r[EMAIL]
+ elif by_username[homeuuid_and_username] != r[EMAIL]:
+ print("ERROR: the username '%s' is listed for both '%s' and '%s' on cluster '%s'" % (r[USERNAME], r[EMAIL], by_username[homeuuid_and_username], r[HOMECLUSTER]))
+ exit(1)
+
+ rows.append(r)
+ return rows
+
+def update_username(args, email, user_uuid, username, migratecluster, migratearv):
+ print("(%s) Updating username of %s to '%s' on %s" % (email, user_uuid, username, migratecluster))
+ if not args.dry_run:
+ try:
+ conflicts = migratearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ migratearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
+ migratearv.users().update(uuid=user_uuid, body={"user": {"username": username}}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, user_uuid, username, migratecluster, e))
+
+
+def choose_new_user(args, by_email, email, userhome, username, old_user_uuid, clusters):
+ candidates = []
+ conflict = False
+ for b in by_email[email].values():
+ if b[2].startswith(userhome):
+ candidates.append(b)
+ if b[1] != username and b[3] == userhome:
+ print("(%s) Cannot migrate %s, conflicting usernames %s and %s" % (email, old_user_uuid, b[1], username))
+ conflict = True
+ break
+ if conflict:
+ return None
+ if len(candidates) == 0:
+ if len(userhome) == 5 and userhome not in clusters:
+ print("(%s) Cannot migrate %s, unknown home cluster %s (typo?)" % (email, old_user_uuid, userhome))
+ return None
+ print("(%s) No user listed with same email to migrate %s to %s, will create new user with username '%s'" % (email, old_user_uuid, userhome, username))
+ if not args.dry_run:
+ newhomecluster = userhome[0:5]
+ homearv = clusters[userhome]
+ user = None
+ try:
+ conflicts = homearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ homearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
+ user = homearv.users().create(body={"user": {"email": email, "username": username}}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not create user: %s" % (email, str(e)))
+ return None
+
+ tup = (email, username, user["uuid"], userhome)
+ else:
+ # dry run
+ tup = (email, username, "%s-tpzed-xfakexfakexfake" % (userhome[0:5]), userhome)
+ by_email[email][tup[2]] = tup
+ candidates.append(tup)
+ if len(candidates) > 1:
+ print("(%s) Multiple users listed to migrate %s to %s, use full uuid" % (email, old_user_uuid, userhome))
+ return None
+ return candidates[0][2]
+
+
+def activate_remote_user(args, email, homearv, migratearv, old_user_uuid, new_user_uuid):
+ # create a token for the new user and salt it for the
+ # migration cluster, then use it to access the migration
+ # cluster as the new user once before merging to ensure
+ # the new user is known on that cluster.
+ migratecluster = migratearv._rootDesc["uuidPrefix"]
+ try:
+ if not args.dry_run:
+ newtok = homearv.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': new_user_uuid}}).execute()
+ else:
+ newtok = {"uuid": "dry-run", "api_token": "12345"}
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not create API token for %s: %s" % (email, new_user_uuid, e))
+ return None
+
+ salted = 'v2/' + newtok["uuid"] + '/' + hmac.new(newtok["api_token"].encode(),
+ msg=migratecluster.encode(),
+ digestmod='sha1').hexdigest()
+ try:
+ ru = urllib.parse.urlparse(migratearv._rootDesc["rootUrl"])
+ if not args.dry_run:
+ newuser = arvados.api(host=ru.netloc, token=salted, insecure=os.environ.get("ARVADOS_API_HOST_INSECURE")).users().current().execute()
+ else:
+ newuser = {"is_active": True, "username": username}
+ except arvados.errors.ApiError as e:
+ print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
+ return None
+
+ try:
+ olduser = migratearv.users().get(uuid=old_user_uuid).execute()
+ except arvados.errors.ApiError as e:
+ if e.resp.status != 404:
+ print("(%s) Could not retrieve user %s from %s, user may have already been migrated: %s" % (email, old_user_uuid, migratecluster, e))
+ return None
+
+ if not newuser["is_active"]:
+ print("(%s) Activating user %s on %s" % (email, new_user_uuid, migratecluster))
+ try:
+ if not args.dry_run:
+ migratearv.users().update(uuid=new_user_uuid, body={"is_active": True}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not activate user %s on %s: %s" % (email, new_user_uuid, migratecluster, e))
+ return None
+
+ if olduser["is_admin"] and not newuser["is_admin"]:
+ print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
+ return None
+
+ return newuser
+
+def migrate_user(args, migratearv, email, new_user_uuid, old_user_uuid):
+ try:
+ if not args.dry_run:
+ grp = migratearv.groups().create(body={
+ "owner_uuid": new_user_uuid,
+ "name": "Migrated from %s (%s)" % (email, old_user_uuid),
+ "group_class": "project"
+ }, ensure_unique_name=True).execute()
+ migratearv.users().merge(old_user_uuid=old_user_uuid,
+ new_user_uuid=new_user_uuid,
+ new_owner_uuid=grp["uuid"],
+ redirect_to_new_user=True).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error migrating user: %s" % (email, e))
+
+
+def main():
+
+ parser = argparse.ArgumentParser(description='Migrate users to federated identity, see https://doc.arvados.org/admin/merge-remote-account.html')
+ parser.add_argument(
+ '--version', action='version', version="%s %s" % (sys.argv[0], __version__),
+ help='Print version and exit.')
+ parser.add_argument('--tokens', type=str, required=False)
+ group = parser.add_mutually_exclusive_group(required=True)
+ group.add_argument('--report', type=str, help="Generate report .csv file listing users by email address and their associated Arvados accounts")
+ group.add_argument('--migrate', type=str, help="Consume report .csv and migrate users to designated Arvados accounts")
+ group.add_argument('--dry-run', type=str, help="Consume report .csv and report how user would be migrated to designated Arvados accounts")
+ group.add_argument('--check', action="store_true", help="Check that tokens are usable and the federation is well connected")
+ args = parser.parse_args()
+
+ clusters, errors, loginCluster = connect_clusters(args)
+
+ if errors:
+ for e in errors:
+ print("ERROR: "+str(e))
+ exit(1)
+
+ if args.check:
+ print("Tokens file passed checks")
+ exit(0)
+
+ rows, by_email, by_username = fetch_users(clusters, loginCluster)
if args.report:
out = csv.writer(open(args.report, "wt"))
@@ -146,21 +317,13 @@ def main():
if args.dry_run:
print("Performing dry run")
- rows = []
-
- with open(args.migrate or args.dry_run, "rt") as f:
- for r in csv.reader(f):
- if r[0] == "email":
- continue
- by_email.setdefault(r[0], {})
- by_email[r[0]][r[2]] = r
- rows.append(r)
+ rows = read_migrations(args, by_email, by_username)
for r in rows:
- email = r[0]
- username = r[1]
- old_user_uuid = r[2]
- userhome = r[3]
+ email = r[EMAIL]
+ username = r[USERNAME]
+ old_user_uuid = r[UUID]
+ userhome = r[HOMECLUSTER]
if userhome == "":
print("(%s) Skipping %s, no home cluster specified" % (email, old_user_uuid))
@@ -168,137 +331,31 @@ def main():
migratecluster = old_user_uuid[0:5]
migratearv = clusters[migratecluster]
if migratearv.users().get(uuid=old_user_uuid).execute()["username"] != username:
- print("(%s) Updating username of %s to '%s' on %s" % (email, old_user_uuid, username, migratecluster))
- if not args.dry_run:
- try:
- conflicts = migratearv.users().list(filters=[["username", "=", username]]).execute()
- if conflicts["items"]:
- migratearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
- migratearv.users().update(uuid=old_user_uuid, body={"user": {"username": username}}).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, old_user_uuid, username, migratecluster, e))
+ update_username(args, email, old_user_uuid, username, migratecluster, migratearv)
continue
- candidates = []
- conflict = False
- for b in by_email[email].values():
- if b[2].startswith(userhome):
- candidates.append(b)
- if b[1] != username and b[3] == userhome:
- print("(%s) Cannot migrate %s, conflicting usernames %s and %s" % (email, old_user_uuid, b[1], username))
- conflict = True
- break
- if conflict:
- continue
- if len(candidates) == 0:
- if len(userhome) == 5 and userhome not in clusters:
- print("(%s) Cannot migrate %s, unknown home cluster %s (typo?)" % (email, old_user_uuid, userhome))
- continue
- print("(%s) No user listed with same email to migrate %s to %s, will create new user with username '%s'" % (email, old_user_uuid, userhome, username))
- if not args.dry_run:
- newhomecluster = userhome[0:5]
- homearv = clusters[userhome]
- user = None
- try:
- conflicts = homearv.users().list(filters=[["username", "=", username]]).execute()
- if conflicts["items"]:
- homearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
- user = homearv.users().create(body={"user": {"email": email, "username": username}}).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Could not create user: %s" % (email, str(e)))
- continue
-
- tup = (email, username, user["uuid"], userhome)
- else:
- # dry run
- tup = (email, username, "%s-tpzed-xfakexfakexfake" % (userhome[0:5]), userhome)
- by_email[email][tup[2]] = tup
- candidates.append(tup)
- if len(candidates) > 1:
- print("(%s) Multiple users listed to migrate %s to %s, use full uuid" % (email, old_user_uuid, userhome))
+
+ new_user_uuid = choose_new_user(args, by_email, email, userhome, username, old_user_uuid, clusters)
+ if new_user_uuid is None:
continue
- new_user_uuid = candidates[0][2]
# cluster where the migration is happening
- for arv in clusters.values():
- migratecluster = arv._rootDesc["uuidPrefix"]
+ for migratecluster in clusters:
migratearv = clusters[migratecluster]
# the user's new home cluster
newhomecluster = userhome[0:5]
homearv = clusters[newhomecluster]
- # create a token for the new user and salt it for the
- # migration cluster, then use it to access the migration
- # cluster as the new user once before merging to ensure
- # the new user is known on that cluster.
- try:
- if not args.dry_run:
- newtok = homearv.api_client_authorizations().create(body={
- "api_client_authorization": {'owner_uuid': new_user_uuid}}).execute()
- else:
- newtok = {"uuid": "dry-run", "api_token": "12345"}
- except arvados.errors.ApiError as e:
- print("(%s) Could not create API token for %s: %s" % (email, new_user_uuid, e))
- continue
-
- salted = 'v2/' + newtok["uuid"] + '/' + hmac.new(newtok["api_token"].encode(),
- msg=migratecluster.encode(),
- digestmod='sha1').hexdigest()
- try:
- ru = urllib.parse.urlparse(migratearv._rootDesc["rootUrl"])
- if not args.dry_run:
- newuser = arvados.api(host=ru.netloc, token=salted, insecure=os.environ.get("ARVADOS_API_HOST_INSECURE")).users().current().execute()
- else:
- newuser = {"is_active": True, "username": username}
- except arvados.errors.ApiError as e:
- print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
- continue
-
- try:
- olduser = migratearv.users().get(uuid=old_user_uuid).execute()
- except arvados.errors.ApiError as e:
- if e.resp.status != 404:
- print("(%s) Could not retrieve user %s from %s, user may have already been migrated: %s" % (email, old_user_uuid, migratecluster, e))
- continue
-
- if not newuser["is_active"]:
- print("(%s) Activating user %s on %s" % (email, new_user_uuid, migratecluster))
- try:
- if not args.dry_run:
- migratearv.users().update(uuid=new_user_uuid, body={"is_active": True}).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Could not activate user %s on %s: %s" % (email, new_user_uuid, migratecluster, e))
- continue
-
- if olduser["is_admin"] and not newuser["is_admin"]:
- print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
+ newuser = activate_remote_user(args, email, homearv, migratearv, old_user_uuid, new_user_uuid)
+ if newuser is None:
continue
print("(%s) Migrating %s to %s on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
- try:
- if not args.dry_run:
- grp = migratearv.groups().create(body={
- "owner_uuid": new_user_uuid,
- "name": "Migrated from %s (%s)" % (email, old_user_uuid),
- "group_class": "project"
- }, ensure_unique_name=True).execute()
- migratearv.users().merge(old_user_uuid=old_user_uuid,
- new_user_uuid=new_user_uuid,
- new_owner_uuid=grp["uuid"],
- redirect_to_new_user=True).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Error migrating user: %s" % (email, e))
+ migrate_user(args, migratearv, email, new_user_uuid, old_user_uuid)
if newuser['username'] != username:
- try:
- if not args.dry_run:
- conflicts = migratearv.users().list(filters=[["username", "=", username]]).execute()
- if conflicts["items"]:
- migratearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
- migratearv.users().update(uuid=new_user_uuid, body={"user": {"username": username}}).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, new_user_uuid, username, migratecluster, e))
+ update_username(args, email, new_user_uuid, username, migratecluster, migratearv)
if __name__ == "__main__":
main()
diff --git a/sdk/python/tests/fed-migrate/run-test.cwl b/sdk/python/tests/fed-migrate/run-test.cwl
deleted file mode 100644
index 623a9c11f..000000000
--- a/sdk/python/tests/fed-migrate/run-test.cwl
+++ /dev/null
@@ -1,291 +0,0 @@
-#!/usr/bin/env cwl-runner
-class: Workflow
-cwlVersion: v1.0
-id: '#main'
-inputs:
- - id: arvados_api_hosts
- type:
- items: string
- type: array
- - id: superuser_tokens
- type:
- items: string
- type: array
- - default: arv-federation-migrate
- id: fed_migrate
- type: string
-outputs:
- - id: report3
- outputSource: main_2/report3
- type: File
-requirements:
- InlineJavascriptRequirement: {}
- MultipleInputFeatureRequirement: {}
- ScatterFeatureRequirement: {}
- StepInputExpressionRequirement: {}
- SubworkflowFeatureRequirement: {}
-steps:
- - id: main_1
- in:
- arvados_api_hosts:
- source: arvados_api_hosts
- create_users:
- default:
- class: File
- location: create_users.py
- superuser_tokens:
- source: superuser_tokens
- out:
- - ar
- run:
- arguments:
- - python
- - $(inputs.create_users)
- - _script
- class: CommandLineTool
- id: main_1_embed
- inputs:
- - id: arvados_api_hosts
- type:
- items: string
- type: array
- - id: superuser_tokens
- type:
- items: string
- type: array
- - id: create_users
- type: File
- outputs:
- - id: ar
- outputBinding:
- outputEval: $(inputs.arvados_api_hosts)
- type:
- items: string
- type: array
- requirements:
- InitialWorkDirRequirement:
- listing:
- - entry: |
- {
- "arvados_api_hosts": $(inputs.arvados_api_hosts),
- "superuser_tokens": $(inputs.superuser_tokens)
- }
- entryname: _script
- InlineJavascriptRequirement: {}
- - id: main_2
- in:
- ar:
- source: main_1/ar
- arvados_api_hosts:
- source: arvados_api_hosts
- fed_migrate:
- source: fed_migrate
- host:
- valueFrom: '$(inputs.arvados_api_hosts[0])'
- superuser_tokens:
- source: superuser_tokens
- token:
- valueFrom: '$(inputs.superuser_tokens[0])'
- out:
- - report
- - report2
- - report3
- - r
- run:
- class: Workflow
- id: main_2_embed
- inputs:
- - id: ar
- type:
- items: string
- type: array
- - id: arvados_api_hosts
- type:
- items: string
- type: array
- - id: superuser_tokens
- type:
- items: string
- type: array
- - id: fed_migrate
- type: string
- - id: host
- type: Any
- - id: token
- type: Any
- outputs:
- - id: report
- outputSource: main_2_embed_1/report
- type: File
- - id: report2
- outputSource: main_2_embed_2/report2
- type: File
- - id: report3
- outputSource: main_2_embed_3/report3
- type: File
- - id: r
- outputSource: main_2_embed_4/r
- type: File
- requirements:
- - class: EnvVarRequirement
- envDef:
- ARVADOS_API_HOST: $(inputs.host)
- ARVADOS_API_TOKEN: $(inputs.token)
- steps:
- - id: main_2_embed_1
- in:
- fed_migrate:
- source: fed_migrate
- host:
- source: host
- token:
- source: token
- out:
- - report
- run:
- arguments:
- - $(inputs.fed_migrate)
- - '--report'
- - report.csv
- class: CommandLineTool
- id: main_2_embed_1_embed
- inputs:
- - id: fed_migrate
- type: string
- - id: host
- type: Any
- - id: token
- type: Any
- outputs:
- - id: report
- outputBinding:
- glob: report.csv
- type: File
- requirements:
- InlineJavascriptRequirement: {}
- - id: main_2_embed_2
- in:
- host:
- source: host
- report:
- source: main_2_embed_1/report
- token:
- source: token
- out:
- - report2
- run:
- arguments:
- - sed
- - '-E'
- - 's/,(case[1-8])2?,/,1,/g'
- class: CommandLineTool
- id: main_2_embed_2_embed
- inputs:
- - id: report
- type: File
- - id: host
- type: Any
- - id: token
- type: Any
- outputs:
- - id: report2
- outputBinding:
- glob: report.csv
- type: File
- requirements:
- InlineJavascriptRequirement: {}
- stdin: $(inputs.report)
- stdout: report.csv
- - id: main_2_embed_3
- in:
- fed_migrate:
- source: fed_migrate
- host:
- source: host
- report2:
- source: main_2_embed_2/report2
- token:
- source: token
- out:
- - report3
- run:
- arguments:
- - $(inputs.fed_migrate)
- - '--migrate'
- - $(inputs.report)
- class: CommandLineTool
- id: main_2_embed_3_embed
- inputs:
- - id: report2
- type: File
- - id: fed_migrate
- type: string
- - id: host
- type: Any
- - id: token
- type: Any
- outputs:
- - id: report3
- outputBinding:
- outputEval: $(inputs.report2)
- type: File
- requirements:
- InlineJavascriptRequirement: {}
- - id: main_2_embed_4
- in:
- arvados_api_hosts:
- source: arvados_api_hosts
- check:
- default:
- class: File
- location: check.py
- host:
- source: host
- report3:
- source: main_2_embed_3/report3
- superuser_tokens:
- source: superuser_tokens
- token:
- source: token
- out:
- - r
- run:
- arguments:
- - python
- - $(inputs.check)
- - _script
- class: CommandLineTool
- id: main_2_embed_4_embed
- inputs:
- - id: report3
- type: File
- - id: host
- type: Any
- - id: token
- type: Any
- - id: arvados_api_hosts
- type:
- items: string
- type: array
- - id: superuser_tokens
- type:
- items: string
- type: array
- - id: check
- type: File
- outputs:
- - id: r
- outputBinding:
- outputEval: $(inputs.report3)
- type: File
- requirements:
- InitialWorkDirRequirement:
- listing:
- - entry: |
- {
- "arvados_api_hosts": $(inputs.arvados_api_hosts),
- "superuser_tokens": $(inputs.superuser_tokens)
- }
- entryname: _script
- InlineJavascriptRequirement: {}
-
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 2999d3193..bdbd5fa23 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -649,6 +649,7 @@ EOF
echo "rebuild <config> build arvbox Docker image, no layer cache"
echo "checkpoint create database backup"
echo "restore restore checkpoint"
+ echo "hotreset reset database and restart API without restarting container"
echo "reset delete arvbox arvados data (be careful!)"
echo "destroy delete all arvbox code and data (be careful!)"
echo "log <service> tail log of specified service"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list