[ARVADOS] created: f074ef8181c2b93cbfcd870eb67c8c95cb0e442a

Git user git at public.curoverse.com
Tue Mar 14 12:31:01 EDT 2017


        at  f074ef8181c2b93cbfcd870eb67c8c95cb0e442a (commit)


commit f074ef8181c2b93cbfcd870eb67c8c95cb0e442a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Mar 9 17:41:55 2017 -0500

    8567: Fix migrate links to use PDH instead of UUID.
    
    Better error reporting.
    
    Migrate script cleans up /var/lib/docker inside container.

diff --git a/docker/migrate-docker19/migrate.sh b/docker/migrate-docker19/migrate.sh
index 63d7274..58d6665 100755
--- a/docker/migrate-docker19/migrate.sh
+++ b/docker/migrate-docker19/migrate.sh
@@ -2,6 +2,14 @@
 
 set -e
 
+function cleanup {
+    kill $(cat /var/run/docker.pid)
+    sleep 1
+    rm -rf /var/lib/docker/*
+}
+
+trap cleanup EXIT
+
 /root/dnd.sh &
 sleep 2
 
@@ -28,11 +36,6 @@ sleep 2
 
 docker images -a
 
-UUID=$(arv-keepdocker --project-uuid=$project_uuid $image_repo $image_tag)
-
-kill $(cat /var/run/docker.pid)
-sleep 1
-
-chmod ugo+rwx -R /var/lib/docker
+UUID=$(arv-keepdocker --force-image-format --project-uuid=$project_uuid $image_repo $image_tag)
 
 echo "Migrated uuid is $UUID"
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 569b315..5603ade 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -393,16 +393,26 @@ def migrate19():
     for m in migration_links:
         already_migrated.add(m["tail_uuid"])
 
-    need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
+    items = arvados.util.list_all(api_client.collections().list,
+                                  filters=[["uuid", "in", [img["collection"] for img in old_images]]],
+                                  select=["uuid", "portable_data_hash"])
+    uuid_to_pdh = {i["uuid"]: i["portable_data_hash"] for i in items}
+    need_migrate = [img for img in old_images
+                    if uuid_to_pdh[img["collection"]] not in already_migrated]
 
     logger.info("Already migrated %i images", len(already_migrated))
     logger.info("Need to migrate %i images", len(need_migrate))
 
+    success = []
+    failures = []
     for old_image in need_migrate:
-        logger.info("Migrating %s", old_image["collection"])
+        if uuid_to_pdh[old_image["collection"]] in already_migrated:
+            continue
+
+        logger.info("Migrating %s:%s (%s)", old_image["repo"], old_image["tag"], old_image["collection"])
 
-        col = CollectionReader(old_image["collection"])
-        tarfile = col.keys()[0]
+        oldcol = CollectionReader(old_image["collection"])
+        tarfile = oldcol.keys()[0]
 
         try:
             varlibdocker = tempfile.mkdtemp()
@@ -424,26 +434,37 @@ def migrate19():
                              tarfile[0:40],
                              old_image["repo"],
                              old_image["tag"],
-                             col.api_response()["owner_uuid"]]
+                             oldcol.api_response()["owner_uuid"]]
 
                 out = subprocess.check_output(dockercmd)
 
-            new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
-            api_client.links().create(body={"link": {
-                'owner_uuid': col.api_response()["owner_uuid"],
-                'link_class': arvados.commands.keepdocker._migration_link_class,
-                'name': arvados.commands.keepdocker._migration_link_name,
-                'tail_uuid': old_image["collection"],
-                'head_uuid': new_collection.group(1)
-                }}).execute(num_retries=3)
-
-            logger.info("Migrated '%s' to '%s'", old_image["collection"], new_collection.group(1))
+            migrated = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
+            if migrated:
+                newcol = CollectionReader(migrated.group(1))
+
+                api_client.links().create(body={"link": {
+                    'owner_uuid': oldcol.api_response()["owner_uuid"],
+                    'link_class': arvados.commands.keepdocker._migration_link_class,
+                    'name': arvados.commands.keepdocker._migration_link_name,
+                    'tail_uuid': oldcol.portable_data_hash(),
+                    'head_uuid': newcol.portable_data_hash()
+                    }}).execute(num_retries=3)
+
+                logger.info("Migrated '%s' to '%s'", oldcol.portable_data_hash(), newcol.portable_data_hash())
+                already_migrated.add(oldcol.portable_data_hash())
+                success.append(old_image["collection"])
+            else:
+                logger.error("Error migrating '%s'", old_image["collection"])
+                failures.append(old_image["collection"])
         except Exception as e:
             logger.exception("Migration failed")
+            failures.append(old_image["collection"])
         finally:
             shutil.rmtree(varlibdocker)
 
-    logger.info("All done")
+    logger.info("Successfully migrated %i images", len(success))
+    if failures:
+        logger.error("Failure migrating images: %s", failures)
 
 
 def main(arguments=None, stdout=sys.stdout):

commit 2ed5325b37efa7aa9d38a60c1d5c8b4980df8489
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Mar 7 10:02:39 2017 -0500

    8567: Add check for ARVADOS_API_HOST_INSECURE

diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 9f7cd79..569b315 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -409,7 +409,8 @@ def migrate19():
             with tempfile.NamedTemporaryFile() as envfile:
                 envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
                 envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
-                envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+                if "ARVADOS_API_HOST_INSECURE" in os.environ:
+                    envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
                 envfile.flush()
 
                 dockercmd = ["docker", "run",

commit 55c719bff1b34d037506639fd4cf7f0a74f4c3cb
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Mar 7 09:26:17 2017 -0500

    8567: Rename docker19-migrate to migrate-docker19 for consistency with
    arv-migrate-docker19.  Add docstrict to migrate19() function.

diff --git a/docker/docker19-migrate/build.sh b/docker/docker19-migrate/build.sh
deleted file mode 100755
index e6dc998..0000000
--- a/docker/docker19-migrate/build.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-exec docker build -t arvados/docker19-migrate .
diff --git a/docker/docker19-migrate/Dockerfile b/docker/migrate-docker19/Dockerfile
similarity index 100%
rename from docker/docker19-migrate/Dockerfile
rename to docker/migrate-docker19/Dockerfile
diff --git a/docker/docker19-migrate/README b/docker/migrate-docker19/README
similarity index 88%
rename from docker/docker19-migrate/README
rename to docker/migrate-docker19/README
index 59fa626..3fcd07a 100644
--- a/docker/docker19-migrate/README
+++ b/docker/migrate-docker19/README
@@ -7,7 +7,8 @@ Requires Docker running on the local host.
 
 Usage:
 
-1) Run ./build.sh to create arvados/docker19-migrate Docker image.
+1) Run arvados/docker/migrate-docker19/build.sh to create
+arvados/migrate-docker19 Docker image.
 
 2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate.
 
diff --git a/docker/migrate-docker19/build.sh b/docker/migrate-docker19/build.sh
new file mode 100755
index 0000000..3a36cd4
--- /dev/null
+++ b/docker/migrate-docker19/build.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec docker build -t arvados/migrate-docker19 .
diff --git a/docker/docker19-migrate/dnd.sh b/docker/migrate-docker19/dnd.sh
similarity index 100%
rename from docker/docker19-migrate/dnd.sh
rename to docker/migrate-docker19/dnd.sh
diff --git a/docker/docker19-migrate/migrate.sh b/docker/migrate-docker19/migrate.sh
similarity index 100%
rename from docker/docker19-migrate/migrate.sh
rename to docker/migrate-docker19/migrate.sh
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 7e1b170..9f7cd79 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -342,6 +342,34 @@ _migration_link_class = 'docker_image_migration'
 _migration_link_name = 'migrate_1.9_1.10'
 
 def migrate19():
+    """Docker image format migration tool for Arvados.
+
+    This converts Docker images stored in Arvados from image format v1
+    (Docker <= 1.9) to image format v2 (Docker >= 1.10).
+
+    Requires Docker running on the local host.
+
+    Usage:
+
+    1) Run arvados/docker/migrate-docker19/build.sh to create
+    arvados/migrate-docker19 Docker image.
+
+    2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate.
+
+    3) Run arv-migrate-docker19
+
+    This will query Arvados for v1 format Docker images.  For each image that
+    does not already have a corresponding v2 format image (as indicated by a
+    docker_image_migration tag) it will perform the following process:
+
+    i) download the image from Arvados
+    ii) load it into Docker
+    iii) update the Docker version, which updates the image
+    iv) save the v2 format image and upload to Arvados
+    v) create a migration link
+
+    """
+
     api_client  = arvados.api()
 
     images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
@@ -389,7 +417,7 @@ def migrate19():
                              "--rm",
                              "--env-file", envfile.name,
                              "--volume", "%s:/var/lib/docker" % varlibdocker,
-                             "arvados/docker19-migrate",
+                             "arvados/migrate-docker19",
                              "/root/migrate.sh",
                              "%s/%s" % (old_image["collection"], tarfile),
                              tarfile[0:40],

commit 6058f336a4562b6d6780c4fcfc4d28b6b2a51948
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Mar 6 14:52:55 2017 -0500

    8567: Move out of tools/ into sdk/python and docker/docker19-migrate.

diff --git a/tools/docker-migrator/docker/Dockerfile b/docker/docker19-migrate/Dockerfile
similarity index 100%
rename from tools/docker-migrator/docker/Dockerfile
rename to docker/docker19-migrate/Dockerfile
diff --git a/tools/docker-migrator/README b/docker/docker19-migrate/README
similarity index 96%
rename from tools/docker-migrator/README
rename to docker/docker19-migrate/README
index d94eb88..59fa626 100644
--- a/tools/docker-migrator/README
+++ b/docker/docker19-migrate/README
@@ -11,7 +11,7 @@ Usage:
 
 2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate.
 
-3) Run ./migrate.py
+3) Run arv-migrate-docker19
 
 This will query Arvados for v1 format Docker images.  For each image that does
 not already have a corresponding v2 format image (as indicated by a
diff --git a/docker/docker19-migrate/build.sh b/docker/docker19-migrate/build.sh
new file mode 100755
index 0000000..e6dc998
--- /dev/null
+++ b/docker/docker19-migrate/build.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec docker build -t arvados/docker19-migrate .
diff --git a/tools/docker-migrator/docker/dnd.sh b/docker/docker19-migrate/dnd.sh
similarity index 100%
rename from tools/docker-migrator/docker/dnd.sh
rename to docker/docker19-migrate/dnd.sh
diff --git a/tools/docker-migrator/docker/migrate.sh b/docker/docker19-migrate/migrate.sh
similarity index 100%
rename from tools/docker-migrator/docker/migrate.sh
rename to docker/docker19-migrate/migrate.sh
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 3ffb7f3..7e1b170 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -11,6 +11,7 @@ import subprocess
 import sys
 import tarfile
 import tempfile
+import shutil
 import _strptime
 
 from operator import itemgetter
@@ -20,10 +21,17 @@ import arvados
 import arvados.util
 import arvados.commands._util as arv_cmd
 import arvados.commands.put as arv_put
+from arvados.collection import CollectionReader
 import ciso8601
+import logging
+import arvados.config
 
 from arvados._version import __version__
 
+logger = logging.getLogger('arvados.keepdocker')
+logger.setLevel(logging.DEBUG if arvados.config.get('ARVADOS_DEBUG')
+                else logging.INFO)
+
 EARLIEST_DATETIME = datetime.datetime(datetime.MINYEAR, 1, 1, 0, 0, 0)
 STAT_CACHE_ERRORS = (IOError, OSError, ValueError)
 
@@ -103,15 +111,15 @@ def docker_image_format(image_hash):
 def docker_image_compatible(api, image_hash):
     supported = api._rootDesc.get('dockerImageFormats', [])
     if not supported:
-        print >>sys.stderr, "arv-keepdocker: warning: server does not specify supported image formats (see docker_image_formats in server config). Continuing."
+        logger.warn("server does not specify supported image formats (see docker_image_formats in server config). Continuing.")
         return True
 
     fmt = docker_image_format(image_hash)
     if fmt in supported:
         return True
     else:
-        print >>sys.stderr, "arv-keepdocker: image format is {!r} " \
-            "but server supports only {!r}".format(fmt, supported)
+        logger.error("image format is {!r} " \
+            "but server supports only {!r}".format(fmt, supported))
         return False
 
 def docker_images():
@@ -332,59 +340,82 @@ def _uuid2pdh(api, uuid):
 
 _migration_link_class = 'docker_image_migration'
 _migration_link_name = 'migrate_1.9_1.10'
-def _migrate19_link(api, root_uuid, old_uuid, new_uuid):
-    old_pdh = _uuid2pdh(api, old_uuid)
-    new_pdh = _uuid2pdh(api, new_uuid)
-    if not api.links().list(filters=[
-            ['owner_uuid', '=', root_uuid],
-            ['link_class', '=', _migration_link_class],
-            ['name', '=', _migration_link_name],
-            ['tail_uuid', '=', old_pdh],
-            ['head_uuid', '=', new_pdh]]).execute()['items']:
-        print >>sys.stderr, 'Creating migration link {} -> {}: '.format(
-            old_pdh, new_pdh),
-        link = api.links().create(body={
-            'owner_uuid': root_uuid,
-            'link_class': _migration_link_class,
-            'name': _migration_link_name,
-            'tail_uuid': old_pdh,
-            'head_uuid': new_pdh,
-        }).execute()
-        print >>sys.stderr, '{}'.format(link['uuid'])
-        return link
 
 def migrate19():
-    api = arvados.api('v1')
-    user = api.users().current().execute()
-    if not user['is_admin']:
-        raise Exception("This command requires an admin token")
-    root_uuid = user['uuid'][:12] + '000000000000000'
-    new_image_uuids = {}
-    images = list_images_in_arv(api, 2)
+    api_client  = arvados.api()
+
+    images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
+
     is_new = lambda img: img['dockerhash'].startswith('sha256:')
 
     count_new = 0
+    old_images = []
     for uuid, img in images:
-        if not re.match(r'^[0-9a-f]{64}$', img["tag"]):
+        if img["dockerhash"].startswith("sha256:"):
             continue
-        key = (img["repo"], img["tag"])
-        if is_new(img) and key not in new_image_uuids:
-            count_new += 1
-            new_image_uuids[key] = uuid
+        key = (img["repo"], img["tag"], img["timestamp"])
+        old_images.append(img)
 
-    count_migrations = 0
-    new_links = []
-    for uuid, img in images:
-        key = (img['repo'], img['tag'])
-        if not is_new(img) and key in new_image_uuids:
-            count_migrations += 1
-            link = _migrate19_link(api, root_uuid, uuid, new_image_uuids[key])
-            if link:
-                new_links.append(link)
+    migration_links = arvados.util.list_all(api_client.links().list, filters=[
+        ['link_class', '=', _migration_link_class],
+        ['name', '=', _migration_link_name],
+    ])
+
+    already_migrated = set()
+    for m in migration_links:
+        already_migrated.add(m["tail_uuid"])
+
+    need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
+
+    logger.info("Already migrated %i images", len(already_migrated))
+    logger.info("Need to migrate %i images", len(need_migrate))
+
+    for old_image in need_migrate:
+        logger.info("Migrating %s", old_image["collection"])
+
+        col = CollectionReader(old_image["collection"])
+        tarfile = col.keys()[0]
+
+        try:
+            varlibdocker = tempfile.mkdtemp()
+            with tempfile.NamedTemporaryFile() as envfile:
+                envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
+                envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
+                envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+                envfile.flush()
+
+                dockercmd = ["docker", "run",
+                             "--privileged",
+                             "--rm",
+                             "--env-file", envfile.name,
+                             "--volume", "%s:/var/lib/docker" % varlibdocker,
+                             "arvados/docker19-migrate",
+                             "/root/migrate.sh",
+                             "%s/%s" % (old_image["collection"], tarfile),
+                             tarfile[0:40],
+                             old_image["repo"],
+                             old_image["tag"],
+                             col.api_response()["owner_uuid"]]
+
+                out = subprocess.check_output(dockercmd)
+
+            new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
+            api_client.links().create(body={"link": {
+                'owner_uuid': col.api_response()["owner_uuid"],
+                'link_class': arvados.commands.keepdocker._migration_link_class,
+                'name': arvados.commands.keepdocker._migration_link_name,
+                'tail_uuid': old_image["collection"],
+                'head_uuid': new_collection.group(1)
+                }}).execute(num_retries=3)
+
+            logger.info("Migrated '%s' to '%s'", old_image["collection"], new_collection.group(1))
+        except Exception as e:
+            logger.exception("Migration failed")
+        finally:
+            shutil.rmtree(varlibdocker)
+
+    logger.info("All done")
 
-    print >>sys.stderr, "=== {} new-format images, {} migrations detected, " \
-        "{} links added.".format(count_new, count_migrations, len(new_links))
-    return new_links
 
 def main(arguments=None, stdout=sys.stdout):
     args = arg_parser.parse_args(arguments)
@@ -405,15 +436,15 @@ def main(arguments=None, stdout=sys.stdout):
     try:
         image_hash = find_one_image_hash(args.image, args.tag)
     except DockerError as error:
-        print >>sys.stderr, "arv-keepdocker:", error.message
+        logger.error(error.message)
         sys.exit(1)
 
     if not docker_image_compatible(api, image_hash):
         if args.force_image_format:
-            print >>sys.stderr, "arv-keepdocker: forcing incompatible image"
+            logger.warn("forcing incompatible image")
         else:
-            print >>sys.stderr, "arv-keepdocker: refusing to store " \
-                "incompatible format (use --force-image-format to override)"
+            logger.error("refusing to store " \
+                "incompatible format (use --force-image-format to override)")
             sys.exit(1)
 
     image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None
diff --git a/tools/docker-migrator/build.sh b/tools/docker-migrator/build.sh
deleted file mode 100755
index ecef09d..0000000
--- a/tools/docker-migrator/build.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-exec docker build -t arvados/docker19-migrate docker
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
deleted file mode 100755
index ec10b75..0000000
--- a/tools/docker-migrator/migrate.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env python
-
-import arvados
-import arvados.util
-from arvados.collection import CollectionReader
-import arvados.commands.keepdocker
-import re
-import subprocess
-import os
-import tempfile
-import shutil
-
-from pprint import pprint
-
-def main():
-    api_client  = arvados.api()
-
-    images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
-
-    is_new = lambda img: img['dockerhash'].startswith('sha256:')
-
-    count_new = 0
-    old_images = []
-    for uuid, img in images:
-        if img["dockerhash"].startswith("sha256:"):
-            continue
-        key = (img["repo"], img["tag"], img["timestamp"])
-        old_images.append(img)
-
-    migration_links = arvados.util.list_all(api_client.links().list, filters=[
-        ['link_class', '=', arvados.commands.keepdocker._migration_link_class],
-        ['name', '=', arvados.commands.keepdocker._migration_link_name],
-    ])
-
-    already_migrated = set()
-    for m in migration_links:
-        already_migrated.add(m["tail_uuid"])
-
-    need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
-
-    print "Already migrated %i images" % (len(already_migrated))
-    print "Need to migrate %i images" % (len(need_migrate))
-
-    for old_image in need_migrate:
-        print "Migrating %s" % (old_image["collection"])
-
-        col = CollectionReader(old_image["collection"])
-        tarfile = col.keys()[0]
-
-        try:
-            varlibdocker = tempfile.mkdtemp()
-            with tempfile.NamedTemporaryFile() as envfile:
-                envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
-                envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
-                envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
-                envfile.flush()
-
-                dockercmd = ["docker", "run",
-                             "--privileged",
-                             "--rm",
-                             "--env-file", envfile.name,
-                             "--volume", "%s:/var/lib/docker" % varlibdocker,
-                             "arvados/docker19-migrate",
-                             "/root/migrate.sh",
-                             "%s/%s" % (old_image["collection"], tarfile),
-                             tarfile[0:40],
-                             old_image["repo"],
-                             old_image["tag"],
-                             col.api_response()["owner_uuid"]]
-
-                out = subprocess.check_output(dockercmd)
-
-            new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
-            api_client.links().create(body={"link": {
-                'owner_uuid': col.api_response()["owner_uuid"],
-                'link_class': arvados.commands.keepdocker._migration_link_class,
-                'name': arvados.commands.keepdocker._migration_link_name,
-                'tail_uuid': old_image["collection"],
-                'head_uuid': new_collection.group(1)
-                }}).execute(num_retries=3)
-
-            print "Migrated '%s' to '%s'" % (old_image["collection"], new_collection.group(1))
-        finally:
-            shutil.rmtree(varlibdocker)
-
-    print "All done"
-
-
-main()

commit 1d7577d08b6ca646fcca0e941ce614c37051264e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Mar 6 14:31:17 2017 -0500

    8567: Add status reporting to migrate script.

diff --git a/tools/docker-migrator/README b/tools/docker-migrator/README
new file mode 100644
index 0000000..d94eb88
--- /dev/null
+++ b/tools/docker-migrator/README
@@ -0,0 +1,24 @@
+Docker image format migration tool for Arvados.
+
+This converts Docker images stored in Arvados from image format v1
+(Docker <= 1.9) to image format v2 (Docker >= 1.10).
+
+Requires Docker running on the local host.
+
+Usage:
+
+1) Run ./build.sh to create arvados/docker19-migrate Docker image.
+
+2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate.
+
+3) Run ./migrate.py
+
+This will query Arvados for v1 format Docker images.  For each image that does
+not already have a corresponding v2 format image (as indicated by a
+docker_image_migration tag) it will perform the following process:
+
+i) download the image from Arvados
+ii) load it into Docker
+iii) update the Docker version, which updates the image
+iv) save the v2 format image and upload to Arvados
+v) create a migration link
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
index 23b8166..ec10b75 100755
--- a/tools/docker-migrator/migrate.py
+++ b/tools/docker-migrator/migrate.py
@@ -36,31 +36,39 @@ def main():
     for m in migration_links:
         already_migrated.add(m["tail_uuid"])
 
-    for old_image in old_images:
-        if old_image["collection"] in already_migrated:
-            continue
+    need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
+
+    print "Already migrated %i images" % (len(already_migrated))
+    print "Need to migrate %i images" % (len(need_migrate))
+
+    for old_image in need_migrate:
+        print "Migrating %s" % (old_image["collection"])
+
         col = CollectionReader(old_image["collection"])
         tarfile = col.keys()[0]
 
-        varlibdocker = tempfile.mkdtemp()
-
         try:
-            dockercmd = ["docker", "run",
-                         "--privileged",
-                         "--rm",
-                         "--env", "ARVADOS_API_HOST=%s" % (os.environ["ARVADOS_API_HOST"]),
-                         "--env", "ARVADOS_API_TOKEN=%s" % (os.environ["ARVADOS_API_TOKEN"]),
-                         "--env", "ARVADOS_API_HOST_INSECURE=%s" % (os.environ["ARVADOS_API_HOST_INSECURE"]),
-                         "--volume", "%s:/var/lib/docker" % varlibdocker,
-                         "arvados/docker19-migrate",
-                         "/root/migrate.sh",
-                         "%s/%s" % (old_image["collection"], tarfile),
-                         tarfile[0:40],
-                         old_image["repo"],
-                         old_image["tag"],
-                         col.api_response()["owner_uuid"]]
-
-            out = subprocess.check_output(dockercmd)
+            varlibdocker = tempfile.mkdtemp()
+            with tempfile.NamedTemporaryFile() as envfile:
+                envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
+                envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
+                envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+                envfile.flush()
+
+                dockercmd = ["docker", "run",
+                             "--privileged",
+                             "--rm",
+                             "--env-file", envfile.name,
+                             "--volume", "%s:/var/lib/docker" % varlibdocker,
+                             "arvados/docker19-migrate",
+                             "/root/migrate.sh",
+                             "%s/%s" % (old_image["collection"], tarfile),
+                             tarfile[0:40],
+                             old_image["repo"],
+                             old_image["tag"],
+                             col.api_response()["owner_uuid"]]
+
+                out = subprocess.check_output(dockercmd)
 
             new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
             api_client.links().create(body={"link": {
@@ -75,5 +83,7 @@ def main():
         finally:
             shutil.rmtree(varlibdocker)
 
+    print "All done"
+
 
 main()

commit 205b07766565f3ce7b80bcdfee88f83af451fb01
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Mar 6 10:39:57 2017 -0500

    8567: Creates migration links.

diff --git a/tools/docker-migrator/docker/migrate.sh b/tools/docker-migrator/docker/migrate.sh
index 5638950..63d7274 100755
--- a/tools/docker-migrator/docker/migrate.sh
+++ b/tools/docker-migrator/docker/migrate.sh
@@ -2,13 +2,18 @@
 
 set -e
 
-#/root/dnd.sh &
-/migrator/dnd.sh &
+/root/dnd.sh &
 sleep 2
 
-arv-get $1 | docker load
+image_tar_keepref=$1
+image_id=$2
+image_repo=$3
+image_tag=$4
+project_uuid=$5
 
-docker tag $2 $3:$4
+arv-get $image_tar_keepref | docker load
+
+docker tag $image_id $image_repo:$image_tag
 
 docker images -a
 
@@ -18,12 +23,12 @@ sleep 1
 cd /root/pkgs
 dpkg -i libltdl7_2.4.2-1.11+b1_amd64.deb  docker-engine_1.13.1-0~debian-jessie_amd64.deb
 
-/migrator/dnd.sh &
+/root/dnd.sh &
 sleep 2
 
 docker images -a
 
-UUID=$(arv-keepdocker --project-uuid=$5 $3 $4)
+UUID=$(arv-keepdocker --project-uuid=$project_uuid $image_repo $image_tag)
 
 kill $(cat /var/run/docker.pid)
 sleep 1
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
index 4fd7dd6..23b8166 100755
--- a/tools/docker-migrator/migrate.py
+++ b/tools/docker-migrator/migrate.py
@@ -27,8 +27,6 @@ def main():
         key = (img["repo"], img["tag"], img["timestamp"])
         old_images.append(img)
 
-    print old_images
-
     migration_links = arvados.util.list_all(api_client.links().list, filters=[
         ['link_class', '=', arvados.commands.keepdocker._migration_link_class],
         ['name', '=', arvados.commands.keepdocker._migration_link_name],
@@ -38,12 +36,6 @@ def main():
     for m in migration_links:
         already_migrated.add(m["tail_uuid"])
 
-#            ['tail_uuid', '=', old_pdh],
-#            ['head_uuid', '=', new_pdh]]).execute()['items']
-
-    pprint(old_images)
-    pprint(already_migrated)
-
     for old_image in old_images:
         if old_image["collection"] in already_migrated:
             continue
@@ -66,12 +58,20 @@ def main():
                          tarfile[0:40],
                          old_image["repo"],
                          old_image["tag"],
-                         old_image["owner_uuid"]]
+                         col.api_response()["owner_uuid"]]
 
             out = subprocess.check_output(dockercmd)
 
             new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
-            print "New collection is '%s'" % new_collection.group(1)
+            api_client.links().create(body={"link": {
+                'owner_uuid': col.api_response()["owner_uuid"],
+                'link_class': arvados.commands.keepdocker._migration_link_class,
+                'name': arvados.commands.keepdocker._migration_link_name,
+                'tail_uuid': old_image["collection"],
+                'head_uuid': new_collection.group(1)
+                }}).execute(num_retries=3)
+
+            print "Migrated '%s' to '%s'" % (old_image["collection"], new_collection.group(1))
         finally:
             shutil.rmtree(varlibdocker)
 

commit 0eb521827f6a64e567722527f36f0b4d130af504
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Mar 6 10:22:52 2017 -0500

    8567: Docker image migration WIP.

diff --git a/tools/docker-migrator/build.sh b/tools/docker-migrator/build.sh
new file mode 100755
index 0000000..ecef09d
--- /dev/null
+++ b/tools/docker-migrator/build.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec docker build -t arvados/docker19-migrate docker
diff --git a/tools/docker-migrator/docker/Dockerfile b/tools/docker-migrator/docker/Dockerfile
new file mode 100644
index 0000000..7e921c8
--- /dev/null
+++ b/tools/docker-migrator/docker/Dockerfile
@@ -0,0 +1,31 @@
+FROM debian:8
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7 && \
+    gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+    apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D || \
+    apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+
+VOLUME /var/lib/docker
+
+RUN mkdir -p /etc/apt/sources.list.d && \
+    echo deb http://apt.arvados.org/ jessie main > /etc/apt/sources.list.d/apt.arvados.org.list && \
+    apt-get clean && \
+    apt-get update && \
+    apt-get install -yq --no-install-recommends -o Acquire::Retries=6 \
+        git curl python-arvados-python-client apt-transport-https ca-certificates && \
+    apt-get clean
+
+RUN echo deb https://apt.dockerproject.org/repo debian-jessie main > /etc/apt/sources.list.d/docker.list && \
+    apt-get update && \
+    apt-get install -yq --no-install-recommends -o Acquire::Retries=6 \
+        docker-engine=1.9.1-0~jessie && \
+    apt-get clean
+
+RUN mkdir /root/pkgs && \
+    cd /root/pkgs && \
+    curl -L -O https://apt.dockerproject.org/repo/pool/main/d/docker-engine/docker-engine_1.13.1-0~debian-jessie_amd64.deb && \
+    curl -L -O http://httpredir.debian.org/debian/pool/main/libt/libtool/libltdl7_2.4.2-1.11+b1_amd64.deb
+
+ADD migrate.sh dnd.sh /root/
diff --git a/tools/docker-migrator/docker/dnd.sh b/tools/docker-migrator/docker/dnd.sh
new file mode 100755
index 0000000..ce72601
--- /dev/null
+++ b/tools/docker-migrator/docker/dnd.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# Taken from https://github.com/jpetazzo/dind
+
+exec 2>&1
+
+# Ensure that all nodes in /dev/mapper correspond to mapped devices currently loaded by the device-mapper kernel driver
+dmsetup mknodes
+
+: {LOG:=stdio}
+
+# First, make sure that cgroups are mounted correctly.
+CGROUP=/sys/fs/cgroup
+[ -d $CGROUP ] || mkdir $CGROUP
+
+if mountpoint -q $CGROUP ; then
+    break
+else
+    mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP
+fi
+
+if ! mountpoint -q $CGROUP ; then
+    echo "Could not find or mount cgroups. Tried /sys/fs/cgroup and /cgroup.  Did you use --privileged?"
+    exit 1
+fi
+
+if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security
+then
+    mount -t securityfs none /sys/kernel/security || {
+        echo "Could not mount /sys/kernel/security."
+        echo "AppArmor detection and --privileged mode might break."
+    }
+fi
+
+# Mount the cgroup hierarchies exactly as they are in the parent system.
+for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
+do
+        [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
+        mountpoint -q $CGROUP/$SUBSYS ||
+                mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
+
+        # The two following sections address a bug which manifests itself
+        # by a cryptic "lxc-start: no ns_cgroup option specified" when
+        # trying to start containers withina container.
+        # The bug seems to appear when the cgroup hierarchies are not
+        # mounted on the exact same directories in the host, and in the
+        # container.
+
+        # Named, control-less cgroups are mounted with "-o name=foo"
+        # (and appear as such under /proc/<pid>/cgroup) but are usually
+        # mounted on a directory named "foo" (without the "name=" prefix).
+        # Systemd and OpenRC (and possibly others) both create such a
+        # cgroup. To avoid the aforementioned bug, we symlink "foo" to
+        # "name=foo". This shouldn't have any adverse effect.
+        echo $SUBSYS | grep -q ^name= && {
+                NAME=$(echo $SUBSYS | sed s/^name=//)
+                ln -s $SUBSYS $CGROUP/$NAME
+        }
+
+        # Likewise, on at least one system, it has been reported that
+        # systemd would mount the CPU and CPU accounting controllers
+        # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu"
+        # but on a directory called "cpu,cpuacct" (note the inversion
+        # in the order of the groups). This tries to work around it.
+        [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct
+done
+
+# Note: as I write those lines, the LXC userland tools cannot setup
+# a "sub-container" properly if the "devices" cgroup is not in its
+# own hierarchy. Let's detect this and issue a warning.
+grep -q :devices: /proc/1/cgroup ||
+	echo "WARNING: the 'devices' cgroup should be in its own hierarchy."
+grep -qw devices /proc/1/cgroup ||
+	echo "WARNING: it looks like the 'devices' cgroup is not mounted."
+
+# Now, close extraneous file descriptors.
+pushd /proc/self/fd >/dev/null
+for FD in *
+do
+	case "$FD" in
+	# Keep stdin/stdout/stderr
+	[012])
+		;;
+	# Nuke everything else
+	*)
+		eval exec "$FD>&-"
+		;;
+	esac
+done
+popd >/dev/null
+
+
+# If a pidfile is still around (for example after a container restart),
+# delete it so that docker can start.
+rm -rf /var/run/docker.pid
+
+read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
+
+if ! docker daemon --storage-driver=overlay $DOCKER_DAEMON_ARGS ; then
+    docker daemon $DOCKER_DAEMON_ARGS
+fi
diff --git a/tools/docker-migrator/docker/migrate.sh b/tools/docker-migrator/docker/migrate.sh
new file mode 100755
index 0000000..5638950
--- /dev/null
+++ b/tools/docker-migrator/docker/migrate.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+#/root/dnd.sh &
+/migrator/dnd.sh &
+sleep 2
+
+arv-get $1 | docker load
+
+docker tag $2 $3:$4
+
+docker images -a
+
+kill $(cat /var/run/docker.pid)
+sleep 1
+
+cd /root/pkgs
+dpkg -i libltdl7_2.4.2-1.11+b1_amd64.deb  docker-engine_1.13.1-0~debian-jessie_amd64.deb
+
+/migrator/dnd.sh &
+sleep 2
+
+docker images -a
+
+UUID=$(arv-keepdocker --project-uuid=$5 $3 $4)
+
+kill $(cat /var/run/docker.pid)
+sleep 1
+
+chmod ugo+rwx -R /var/lib/docker
+
+echo "Migrated uuid is $UUID"
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
new file mode 100755
index 0000000..4fd7dd6
--- /dev/null
+++ b/tools/docker-migrator/migrate.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+import arvados
+import arvados.util
+from arvados.collection import CollectionReader
+import arvados.commands.keepdocker
+import re
+import subprocess
+import os
+import tempfile
+import shutil
+
+from pprint import pprint
+
+def main():
+    api_client  = arvados.api()
+
+    images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
+
+    is_new = lambda img: img['dockerhash'].startswith('sha256:')
+
+    count_new = 0
+    old_images = []
+    for uuid, img in images:
+        if img["dockerhash"].startswith("sha256:"):
+            continue
+        key = (img["repo"], img["tag"], img["timestamp"])
+        old_images.append(img)
+
+    print old_images
+
+    migration_links = arvados.util.list_all(api_client.links().list, filters=[
+        ['link_class', '=', arvados.commands.keepdocker._migration_link_class],
+        ['name', '=', arvados.commands.keepdocker._migration_link_name],
+    ])
+
+    already_migrated = set()
+    for m in migration_links:
+        already_migrated.add(m["tail_uuid"])
+
+#            ['tail_uuid', '=', old_pdh],
+#            ['head_uuid', '=', new_pdh]]).execute()['items']
+
+    pprint(old_images)
+    pprint(already_migrated)
+
+    for old_image in old_images:
+        if old_image["collection"] in already_migrated:
+            continue
+        col = CollectionReader(old_image["collection"])
+        tarfile = col.keys()[0]
+
+        varlibdocker = tempfile.mkdtemp()
+
+        try:
+            dockercmd = ["docker", "run",
+                         "--privileged",
+                         "--rm",
+                         "--env", "ARVADOS_API_HOST=%s" % (os.environ["ARVADOS_API_HOST"]),
+                         "--env", "ARVADOS_API_TOKEN=%s" % (os.environ["ARVADOS_API_TOKEN"]),
+                         "--env", "ARVADOS_API_HOST_INSECURE=%s" % (os.environ["ARVADOS_API_HOST_INSECURE"]),
+                         "--volume", "%s:/var/lib/docker" % varlibdocker,
+                         "arvados/docker19-migrate",
+                         "/root/migrate.sh",
+                         "%s/%s" % (old_image["collection"], tarfile),
+                         tarfile[0:40],
+                         old_image["repo"],
+                         old_image["tag"],
+                         old_image["owner_uuid"]]
+
+            out = subprocess.check_output(dockercmd)
+
+            new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
+            print "New collection is '%s'" % new_collection.group(1)
+        finally:
+            shutil.rmtree(varlibdocker)
+
+
+main()

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list