[ARVADOS] updated: f2c45e2cb1c597caf05a265456c6e3b93e71080c
Git user
git at public.curoverse.com
Mon Mar 6 14:33:11 EST 2017
Summary of changes:
tools/docker-migrator/README | 24 +++++++++++++++++++
tools/docker-migrator/migrate.py | 52 ++++++++++++++++++++++++----------------
2 files changed, 55 insertions(+), 21 deletions(-)
create mode 100644 tools/docker-migrator/README
via f2c45e2cb1c597caf05a265456c6e3b93e71080c (commit)
from 93384273eec4967dd51c4c4e6c2ad76f0cd3861c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f2c45e2cb1c597caf05a265456c6e3b93e71080c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Mar 6 14:31:17 2017 -0500
8567: Add status reporting to migrate script.
diff --git a/tools/docker-migrator/README b/tools/docker-migrator/README
new file mode 100644
index 0000000..d94eb88
--- /dev/null
+++ b/tools/docker-migrator/README
@@ -0,0 +1,24 @@
+Docker image format migration tool for Arvados.
+
+This converts Docker images stored in Arvados from image format v1
+(Docker <= 1.9) to image format v2 (Docker >= 1.10).
+
+Requires Docker running on the local host.
+
+Usage:
+
+1) Run ./build.sh to create arvados/docker19-migrate Docker image.
+
+2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate.
+
+3) Run ./migrate.py
+
+This will query Arvados for v1 format Docker images. For each image that does
+not already have a corresponding v2 format image (as indicated by a
+docker_image_migration tag) it will perform the following process:
+
+i) download the image from Arvados
+ii) load it into Docker
+iii) update the Docker version, which updates the image
+iv) save the v2 format image and upload to Arvados
+v) create a migration link
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
index 23b8166..ec10b75 100755
--- a/tools/docker-migrator/migrate.py
+++ b/tools/docker-migrator/migrate.py
@@ -36,31 +36,39 @@ def main():
for m in migration_links:
already_migrated.add(m["tail_uuid"])
- for old_image in old_images:
- if old_image["collection"] in already_migrated:
- continue
+ need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
+
+ print "Already migrated %i images" % (len(already_migrated))
+ print "Need to migrate %i images" % (len(need_migrate))
+
+ for old_image in need_migrate:
+ print "Migrating %s" % (old_image["collection"])
+
col = CollectionReader(old_image["collection"])
tarfile = col.keys()[0]
- varlibdocker = tempfile.mkdtemp()
-
try:
- dockercmd = ["docker", "run",
- "--privileged",
- "--rm",
- "--env", "ARVADOS_API_HOST=%s" % (os.environ["ARVADOS_API_HOST"]),
- "--env", "ARVADOS_API_TOKEN=%s" % (os.environ["ARVADOS_API_TOKEN"]),
- "--env", "ARVADOS_API_HOST_INSECURE=%s" % (os.environ["ARVADOS_API_HOST_INSECURE"]),
- "--volume", "%s:/var/lib/docker" % varlibdocker,
- "arvados/docker19-migrate",
- "/root/migrate.sh",
- "%s/%s" % (old_image["collection"], tarfile),
- tarfile[0:40],
- old_image["repo"],
- old_image["tag"],
- col.api_response()["owner_uuid"]]
-
- out = subprocess.check_output(dockercmd)
+ varlibdocker = tempfile.mkdtemp()
+ with tempfile.NamedTemporaryFile() as envfile:
+ envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
+ envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
+ envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+ envfile.flush()
+
+ dockercmd = ["docker", "run",
+ "--privileged",
+ "--rm",
+ "--env-file", envfile.name,
+ "--volume", "%s:/var/lib/docker" % varlibdocker,
+ "arvados/docker19-migrate",
+ "/root/migrate.sh",
+ "%s/%s" % (old_image["collection"], tarfile),
+ tarfile[0:40],
+ old_image["repo"],
+ old_image["tag"],
+ col.api_response()["owner_uuid"]]
+
+ out = subprocess.check_output(dockercmd)
new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
api_client.links().create(body={"link": {
@@ -75,5 +83,7 @@ def main():
finally:
shutil.rmtree(varlibdocker)
+ print "All done"
+
main()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list