[ARVADOS] updated: 58342141dd4639094794e058796adaa9e9d02394
Git user
git at public.curoverse.com
Mon Mar 6 14:53:12 EST 2017
Summary of changes:
.../docker => docker/docker19-migrate}/Dockerfile | 0
.../docker19-migrate}/README | 2 +-
docker/docker19-migrate/build.sh | 2 +
.../docker => docker/docker19-migrate}/dnd.sh | 0
.../docker => docker/docker19-migrate}/migrate.sh | 0
sdk/python/arvados/commands/keepdocker.py | 133 +++++++++++++--------
tools/docker-migrator/build.sh | 2 -
tools/docker-migrator/migrate.py | 89 --------------
8 files changed, 85 insertions(+), 143 deletions(-)
rename {tools/docker-migrator/docker => docker/docker19-migrate}/Dockerfile (100%)
rename {tools/docker-migrator => docker/docker19-migrate}/README (96%)
create mode 100755 docker/docker19-migrate/build.sh
rename {tools/docker-migrator/docker => docker/docker19-migrate}/dnd.sh (100%)
rename {tools/docker-migrator/docker => docker/docker19-migrate}/migrate.sh (100%)
delete mode 100755 tools/docker-migrator/build.sh
delete mode 100755 tools/docker-migrator/migrate.py
via 58342141dd4639094794e058796adaa9e9d02394 (commit)
from f2c45e2cb1c597caf05a265456c6e3b93e71080c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 58342141dd4639094794e058796adaa9e9d02394
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Mar 6 14:52:55 2017 -0500
8567: Move out of tools/ into sdk/python and docker/docker19-migrate.
diff --git a/tools/docker-migrator/docker/Dockerfile b/docker/docker19-migrate/Dockerfile
similarity index 100%
rename from tools/docker-migrator/docker/Dockerfile
rename to docker/docker19-migrate/Dockerfile
diff --git a/tools/docker-migrator/README b/docker/docker19-migrate/README
similarity index 96%
rename from tools/docker-migrator/README
rename to docker/docker19-migrate/README
index d94eb88..59fa626 100644
--- a/tools/docker-migrator/README
+++ b/docker/docker19-migrate/README
@@ -11,7 +11,7 @@ Usage:
2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate.
-3) Run ./migrate.py
+3) Run arv-migrate-docker19
This will query Arvados for v1 format Docker images. For each image that does
not already have a corresponding v2 format image (as indicated by a
diff --git a/docker/docker19-migrate/build.sh b/docker/docker19-migrate/build.sh
new file mode 100755
index 0000000..e6dc998
--- /dev/null
+++ b/docker/docker19-migrate/build.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec docker build -t arvados/docker19-migrate .
diff --git a/tools/docker-migrator/docker/dnd.sh b/docker/docker19-migrate/dnd.sh
similarity index 100%
rename from tools/docker-migrator/docker/dnd.sh
rename to docker/docker19-migrate/dnd.sh
diff --git a/tools/docker-migrator/docker/migrate.sh b/docker/docker19-migrate/migrate.sh
similarity index 100%
rename from tools/docker-migrator/docker/migrate.sh
rename to docker/docker19-migrate/migrate.sh
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 3ffb7f3..7e1b170 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -11,6 +11,7 @@ import subprocess
import sys
import tarfile
import tempfile
+import shutil
import _strptime
from operator import itemgetter
@@ -20,10 +21,17 @@ import arvados
import arvados.util
import arvados.commands._util as arv_cmd
import arvados.commands.put as arv_put
+from arvados.collection import CollectionReader
import ciso8601
+import logging
+import arvados.config
from arvados._version import __version__
+logger = logging.getLogger('arvados.keepdocker')
+logger.setLevel(logging.DEBUG if arvados.config.get('ARVADOS_DEBUG')
+ else logging.INFO)
+
EARLIEST_DATETIME = datetime.datetime(datetime.MINYEAR, 1, 1, 0, 0, 0)
STAT_CACHE_ERRORS = (IOError, OSError, ValueError)
@@ -103,15 +111,15 @@ def docker_image_format(image_hash):
def docker_image_compatible(api, image_hash):
supported = api._rootDesc.get('dockerImageFormats', [])
if not supported:
- print >>sys.stderr, "arv-keepdocker: warning: server does not specify supported image formats (see docker_image_formats in server config). Continuing."
+ logger.warn("server does not specify supported image formats (see docker_image_formats in server config). Continuing.")
return True
fmt = docker_image_format(image_hash)
if fmt in supported:
return True
else:
- print >>sys.stderr, "arv-keepdocker: image format is {!r} " \
- "but server supports only {!r}".format(fmt, supported)
+ logger.error("image format is {!r} " \
+ "but server supports only {!r}".format(fmt, supported))
return False
def docker_images():
@@ -332,59 +340,82 @@ def _uuid2pdh(api, uuid):
_migration_link_class = 'docker_image_migration'
_migration_link_name = 'migrate_1.9_1.10'
-def _migrate19_link(api, root_uuid, old_uuid, new_uuid):
- old_pdh = _uuid2pdh(api, old_uuid)
- new_pdh = _uuid2pdh(api, new_uuid)
- if not api.links().list(filters=[
- ['owner_uuid', '=', root_uuid],
- ['link_class', '=', _migration_link_class],
- ['name', '=', _migration_link_name],
- ['tail_uuid', '=', old_pdh],
- ['head_uuid', '=', new_pdh]]).execute()['items']:
- print >>sys.stderr, 'Creating migration link {} -> {}: '.format(
- old_pdh, new_pdh),
- link = api.links().create(body={
- 'owner_uuid': root_uuid,
- 'link_class': _migration_link_class,
- 'name': _migration_link_name,
- 'tail_uuid': old_pdh,
- 'head_uuid': new_pdh,
- }).execute()
- print >>sys.stderr, '{}'.format(link['uuid'])
- return link
def migrate19():
- api = arvados.api('v1')
- user = api.users().current().execute()
- if not user['is_admin']:
- raise Exception("This command requires an admin token")
- root_uuid = user['uuid'][:12] + '000000000000000'
- new_image_uuids = {}
- images = list_images_in_arv(api, 2)
+ api_client = arvados.api()
+
+ images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
+
is_new = lambda img: img['dockerhash'].startswith('sha256:')
count_new = 0
+ old_images = []
for uuid, img in images:
- if not re.match(r'^[0-9a-f]{64}$', img["tag"]):
+ if img["dockerhash"].startswith("sha256:"):
continue
- key = (img["repo"], img["tag"])
- if is_new(img) and key not in new_image_uuids:
- count_new += 1
- new_image_uuids[key] = uuid
+ key = (img["repo"], img["tag"], img["timestamp"])
+ old_images.append(img)
- count_migrations = 0
- new_links = []
- for uuid, img in images:
- key = (img['repo'], img['tag'])
- if not is_new(img) and key in new_image_uuids:
- count_migrations += 1
- link = _migrate19_link(api, root_uuid, uuid, new_image_uuids[key])
- if link:
- new_links.append(link)
+ migration_links = arvados.util.list_all(api_client.links().list, filters=[
+ ['link_class', '=', _migration_link_class],
+ ['name', '=', _migration_link_name],
+ ])
+
+ already_migrated = set()
+ for m in migration_links:
+ already_migrated.add(m["tail_uuid"])
+
+ need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
+
+ logger.info("Already migrated %i images", len(already_migrated))
+ logger.info("Need to migrate %i images", len(need_migrate))
+
+ for old_image in need_migrate:
+ logger.info("Migrating %s", old_image["collection"])
+
+ col = CollectionReader(old_image["collection"])
+ tarfile = col.keys()[0]
+
+ try:
+ varlibdocker = tempfile.mkdtemp()
+ with tempfile.NamedTemporaryFile() as envfile:
+ envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
+ envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
+ envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+ envfile.flush()
+
+ dockercmd = ["docker", "run",
+ "--privileged",
+ "--rm",
+ "--env-file", envfile.name,
+ "--volume", "%s:/var/lib/docker" % varlibdocker,
+ "arvados/docker19-migrate",
+ "/root/migrate.sh",
+ "%s/%s" % (old_image["collection"], tarfile),
+ tarfile[0:40],
+ old_image["repo"],
+ old_image["tag"],
+ col.api_response()["owner_uuid"]]
+
+ out = subprocess.check_output(dockercmd)
+
+ new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
+ api_client.links().create(body={"link": {
+ 'owner_uuid': col.api_response()["owner_uuid"],
+ 'link_class': arvados.commands.keepdocker._migration_link_class,
+ 'name': arvados.commands.keepdocker._migration_link_name,
+ 'tail_uuid': old_image["collection"],
+ 'head_uuid': new_collection.group(1)
+ }}).execute(num_retries=3)
+
+ logger.info("Migrated '%s' to '%s'", old_image["collection"], new_collection.group(1))
+ except Exception as e:
+ logger.exception("Migration failed")
+ finally:
+ shutil.rmtree(varlibdocker)
+
+ logger.info("All done")
- print >>sys.stderr, "=== {} new-format images, {} migrations detected, " \
- "{} links added.".format(count_new, count_migrations, len(new_links))
- return new_links
def main(arguments=None, stdout=sys.stdout):
args = arg_parser.parse_args(arguments)
@@ -405,15 +436,15 @@ def main(arguments=None, stdout=sys.stdout):
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
- print >>sys.stderr, "arv-keepdocker:", error.message
+ logger.error(error.message)
sys.exit(1)
if not docker_image_compatible(api, image_hash):
if args.force_image_format:
- print >>sys.stderr, "arv-keepdocker: forcing incompatible image"
+ logger.warn("forcing incompatible image")
else:
- print >>sys.stderr, "arv-keepdocker: refusing to store " \
- "incompatible format (use --force-image-format to override)"
+ logger.error("refusing to store " \
+ "incompatible format (use --force-image-format to override)")
sys.exit(1)
image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None
diff --git a/tools/docker-migrator/build.sh b/tools/docker-migrator/build.sh
deleted file mode 100755
index ecef09d..0000000
--- a/tools/docker-migrator/build.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-exec docker build -t arvados/docker19-migrate docker
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
deleted file mode 100755
index ec10b75..0000000
--- a/tools/docker-migrator/migrate.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env python
-
-import arvados
-import arvados.util
-from arvados.collection import CollectionReader
-import arvados.commands.keepdocker
-import re
-import subprocess
-import os
-import tempfile
-import shutil
-
-from pprint import pprint
-
-def main():
- api_client = arvados.api()
-
- images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
-
- is_new = lambda img: img['dockerhash'].startswith('sha256:')
-
- count_new = 0
- old_images = []
- for uuid, img in images:
- if img["dockerhash"].startswith("sha256:"):
- continue
- key = (img["repo"], img["tag"], img["timestamp"])
- old_images.append(img)
-
- migration_links = arvados.util.list_all(api_client.links().list, filters=[
- ['link_class', '=', arvados.commands.keepdocker._migration_link_class],
- ['name', '=', arvados.commands.keepdocker._migration_link_name],
- ])
-
- already_migrated = set()
- for m in migration_links:
- already_migrated.add(m["tail_uuid"])
-
- need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
-
- print "Already migrated %i images" % (len(already_migrated))
- print "Need to migrate %i images" % (len(need_migrate))
-
- for old_image in need_migrate:
- print "Migrating %s" % (old_image["collection"])
-
- col = CollectionReader(old_image["collection"])
- tarfile = col.keys()[0]
-
- try:
- varlibdocker = tempfile.mkdtemp()
- with tempfile.NamedTemporaryFile() as envfile:
- envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
- envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
- envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
- envfile.flush()
-
- dockercmd = ["docker", "run",
- "--privileged",
- "--rm",
- "--env-file", envfile.name,
- "--volume", "%s:/var/lib/docker" % varlibdocker,
- "arvados/docker19-migrate",
- "/root/migrate.sh",
- "%s/%s" % (old_image["collection"], tarfile),
- tarfile[0:40],
- old_image["repo"],
- old_image["tag"],
- col.api_response()["owner_uuid"]]
-
- out = subprocess.check_output(dockercmd)
-
- new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
- api_client.links().create(body={"link": {
- 'owner_uuid': col.api_response()["owner_uuid"],
- 'link_class': arvados.commands.keepdocker._migration_link_class,
- 'name': arvados.commands.keepdocker._migration_link_name,
- 'tail_uuid': old_image["collection"],
- 'head_uuid': new_collection.group(1)
- }}).execute(num_retries=3)
-
- print "Migrated '%s' to '%s'" % (old_image["collection"], new_collection.group(1))
- finally:
- shutil.rmtree(varlibdocker)
-
- print "All done"
-
-
-main()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list