[ARVADOS] created: cee1c18cbf6dedf79a11655113347c166f1214ff

Git user git at public.curoverse.com
Mon Mar 6 10:23:12 EST 2017


        at  cee1c18cbf6dedf79a11655113347c166f1214ff (commit)


commit cee1c18cbf6dedf79a11655113347c166f1214ff
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Mar 6 10:22:52 2017 -0500

    8567: Docker image migration WIP.

diff --git a/tools/docker-migrator/build.sh b/tools/docker-migrator/build.sh
new file mode 100755
index 0000000..ecef09d
--- /dev/null
+++ b/tools/docker-migrator/build.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec docker build -t arvados/docker19-migrate docker
diff --git a/tools/docker-migrator/docker/Dockerfile b/tools/docker-migrator/docker/Dockerfile
new file mode 100644
index 0000000..7e921c8
--- /dev/null
+++ b/tools/docker-migrator/docker/Dockerfile
@@ -0,0 +1,31 @@
+FROM debian:8
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7 && \
+    gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+    apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D || \
+    apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+
+VOLUME /var/lib/docker
+
+RUN mkdir -p /etc/apt/sources.list.d && \
+    echo deb http://apt.arvados.org/ jessie main > /etc/apt/sources.list.d/apt.arvados.org.list && \
+    apt-get clean && \
+    apt-get update && \
+    apt-get install -yq --no-install-recommends -o Acquire::Retries=6 \
+        git curl python-arvados-python-client apt-transport-https ca-certificates && \
+    apt-get clean
+
+RUN echo deb https://apt.dockerproject.org/repo debian-jessie main > /etc/apt/sources.list.d/docker.list && \
+    apt-get update && \
+    apt-get install -yq --no-install-recommends -o Acquire::Retries=6 \
+        docker-engine=1.9.1-0~jessie && \
+    apt-get clean
+
+RUN mkdir /root/pkgs && \
+    cd /root/pkgs && \
+    curl -L -O https://apt.dockerproject.org/repo/pool/main/d/docker-engine/docker-engine_1.13.1-0~debian-jessie_amd64.deb && \
+    curl -L -O http://httpredir.debian.org/debian/pool/main/libt/libtool/libltdl7_2.4.2-1.11+b1_amd64.deb
+
+ADD migrate.sh dnd.sh /root/
diff --git a/tools/docker-migrator/docker/dnd.sh b/tools/docker-migrator/docker/dnd.sh
new file mode 100755
index 0000000..ce72601
--- /dev/null
+++ b/tools/docker-migrator/docker/dnd.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# Taken from https://github.com/jpetazzo/dind
+
+exec 2>&1
+
+# Ensure that all nodes in /dev/mapper correspond to mapped devices currently loaded by the device-mapper kernel driver
+dmsetup mknodes
+
+: {LOG:=stdio}
+
+# First, make sure that cgroups are mounted correctly.
+CGROUP=/sys/fs/cgroup
+[ -d $CGROUP ] || mkdir $CGROUP
+
+if mountpoint -q $CGROUP ; then
+    break
+else
+    mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP
+fi
+
+if ! mountpoint -q $CGROUP ; then
+    echo "Could not find or mount cgroups. Tried /sys/fs/cgroup and /cgroup.  Did you use --privileged?"
+    exit 1
+fi
+
+if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security
+then
+    mount -t securityfs none /sys/kernel/security || {
+        echo "Could not mount /sys/kernel/security."
+        echo "AppArmor detection and --privileged mode might break."
+    }
+fi
+
+# Mount the cgroup hierarchies exactly as they are in the parent system.
+for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
+do
+        [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
+        mountpoint -q $CGROUP/$SUBSYS ||
+                mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
+
+        # The two following sections address a bug which manifests itself
+        # by a cryptic "lxc-start: no ns_cgroup option specified" when
+        # trying to start containers withina container.
+        # The bug seems to appear when the cgroup hierarchies are not
+        # mounted on the exact same directories in the host, and in the
+        # container.
+
+        # Named, control-less cgroups are mounted with "-o name=foo"
+        # (and appear as such under /proc/<pid>/cgroup) but are usually
+        # mounted on a directory named "foo" (without the "name=" prefix).
+        # Systemd and OpenRC (and possibly others) both create such a
+        # cgroup. To avoid the aforementioned bug, we symlink "foo" to
+        # "name=foo". This shouldn't have any adverse effect.
+        echo $SUBSYS | grep -q ^name= && {
+                NAME=$(echo $SUBSYS | sed s/^name=//)
+                ln -s $SUBSYS $CGROUP/$NAME
+        }
+
+        # Likewise, on at least one system, it has been reported that
+        # systemd would mount the CPU and CPU accounting controllers
+        # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu"
+        # but on a directory called "cpu,cpuacct" (note the inversion
+        # in the order of the groups). This tries to work around it.
+        [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct
+done
+
+# Note: as I write those lines, the LXC userland tools cannot setup
+# a "sub-container" properly if the "devices" cgroup is not in its
+# own hierarchy. Let's detect this and issue a warning.
+grep -q :devices: /proc/1/cgroup ||
+	echo "WARNING: the 'devices' cgroup should be in its own hierarchy."
+grep -qw devices /proc/1/cgroup ||
+	echo "WARNING: it looks like the 'devices' cgroup is not mounted."
+
+# Now, close extraneous file descriptors.
+pushd /proc/self/fd >/dev/null
+for FD in *
+do
+	case "$FD" in
+	# Keep stdin/stdout/stderr
+	[012])
+		;;
+	# Nuke everything else
+	*)
+		eval exec "$FD>&-"
+		;;
+	esac
+done
+popd >/dev/null
+
+
+# If a pidfile is still around (for example after a container restart),
+# delete it so that docker can start.
+rm -rf /var/run/docker.pid
+
+read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
+
+if ! docker daemon --storage-driver=overlay $DOCKER_DAEMON_ARGS ; then
+    docker daemon $DOCKER_DAEMON_ARGS
+fi
diff --git a/tools/docker-migrator/docker/migrate.sh b/tools/docker-migrator/docker/migrate.sh
new file mode 100755
index 0000000..5638950
--- /dev/null
+++ b/tools/docker-migrator/docker/migrate.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+#/root/dnd.sh &
+/migrator/dnd.sh &
+sleep 2
+
+arv-get $1 | docker load
+
+docker tag $2 $3:$4
+
+docker images -a
+
+kill $(cat /var/run/docker.pid)
+sleep 1
+
+cd /root/pkgs
+dpkg -i libltdl7_2.4.2-1.11+b1_amd64.deb  docker-engine_1.13.1-0~debian-jessie_amd64.deb
+
+/migrator/dnd.sh &
+sleep 2
+
+docker images -a
+
+UUID=$(arv-keepdocker --project-uuid=$5 $3 $4)
+
+kill $(cat /var/run/docker.pid)
+sleep 1
+
+chmod ugo+rwx -R /var/lib/docker
+
+echo "Migrated uuid is $UUID"
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
new file mode 100755
index 0000000..4fd7dd6
--- /dev/null
+++ b/tools/docker-migrator/migrate.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+import arvados
+import arvados.util
+from arvados.collection import CollectionReader
+import arvados.commands.keepdocker
+import re
+import subprocess
+import os
+import tempfile
+import shutil
+
+from pprint import pprint
+
+def main():
+    api_client  = arvados.api()
+
+    images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
+
+    is_new = lambda img: img['dockerhash'].startswith('sha256:')
+
+    count_new = 0
+    old_images = []
+    for uuid, img in images:
+        if img["dockerhash"].startswith("sha256:"):
+            continue
+        key = (img["repo"], img["tag"], img["timestamp"])
+        old_images.append(img)
+
+    print old_images
+
+    migration_links = arvados.util.list_all(api_client.links().list, filters=[
+        ['link_class', '=', arvados.commands.keepdocker._migration_link_class],
+        ['name', '=', arvados.commands.keepdocker._migration_link_name],
+    ])
+
+    already_migrated = set()
+    for m in migration_links:
+        already_migrated.add(m["tail_uuid"])
+
+#            ['tail_uuid', '=', old_pdh],
+#            ['head_uuid', '=', new_pdh]]).execute()['items']
+
+    pprint(old_images)
+    pprint(already_migrated)
+
+    for old_image in old_images:
+        if old_image["collection"] in already_migrated:
+            continue
+        col = CollectionReader(old_image["collection"])
+        tarfile = col.keys()[0]
+
+        varlibdocker = tempfile.mkdtemp()
+
+        try:
+            dockercmd = ["docker", "run",
+                         "--privileged",
+                         "--rm",
+                         "--env", "ARVADOS_API_HOST=%s" % (os.environ["ARVADOS_API_HOST"]),
+                         "--env", "ARVADOS_API_TOKEN=%s" % (os.environ["ARVADOS_API_TOKEN"]),
+                         "--env", "ARVADOS_API_HOST_INSECURE=%s" % (os.environ["ARVADOS_API_HOST_INSECURE"]),
+                         "--volume", "%s:/var/lib/docker" % varlibdocker,
+                         "arvados/docker19-migrate",
+                         "/root/migrate.sh",
+                         "%s/%s" % (old_image["collection"], tarfile),
+                         tarfile[0:40],
+                         old_image["repo"],
+                         old_image["tag"],
+                         old_image["owner_uuid"]]
+
+            out = subprocess.check_output(dockercmd)
+
+            new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
+            print "New collection is '%s'" % new_collection.group(1)
+        finally:
+            shutil.rmtree(varlibdocker)
+
+
+main()

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list