[ARVADOS] created: 09fc7249a62d44d1de1d7757f6e038b96d42e84a
Git user
git at public.curoverse.com
Tue Mar 7 09:18:58 EST 2017
at 09fc7249a62d44d1de1d7757f6e038b96d42e84a (commit)
commit 09fc7249a62d44d1de1d7757f6e038b96d42e84a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Mar 6 14:52:55 2017 -0500
8567: Move out of tools/ into sdk/python and docker/docker19-migrate.
diff --git a/tools/docker-migrator/docker/Dockerfile b/docker/docker19-migrate/Dockerfile
similarity index 100%
rename from tools/docker-migrator/docker/Dockerfile
rename to docker/docker19-migrate/Dockerfile
diff --git a/tools/docker-migrator/README b/docker/docker19-migrate/README
similarity index 96%
rename from tools/docker-migrator/README
rename to docker/docker19-migrate/README
index d94eb88..59fa626 100644
--- a/tools/docker-migrator/README
+++ b/docker/docker19-migrate/README
@@ -11,7 +11,7 @@ Usage:
2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate.
-3) Run ./migrate.py
+3) Run arv-migrate-docker19
This will query Arvados for v1 format Docker images. For each image that does
not already have a corresponding v2 format image (as indicated by a
diff --git a/docker/docker19-migrate/build.sh b/docker/docker19-migrate/build.sh
new file mode 100755
index 0000000..e6dc998
--- /dev/null
+++ b/docker/docker19-migrate/build.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec docker build -t arvados/docker19-migrate .
diff --git a/tools/docker-migrator/docker/dnd.sh b/docker/docker19-migrate/dnd.sh
similarity index 100%
rename from tools/docker-migrator/docker/dnd.sh
rename to docker/docker19-migrate/dnd.sh
diff --git a/tools/docker-migrator/docker/migrate.sh b/docker/docker19-migrate/migrate.sh
similarity index 100%
rename from tools/docker-migrator/docker/migrate.sh
rename to docker/docker19-migrate/migrate.sh
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 3ffb7f3..7e1b170 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -11,6 +11,7 @@ import subprocess
import sys
import tarfile
import tempfile
+import shutil
import _strptime
from operator import itemgetter
@@ -20,10 +21,17 @@ import arvados
import arvados.util
import arvados.commands._util as arv_cmd
import arvados.commands.put as arv_put
+from arvados.collection import CollectionReader
import ciso8601
+import logging
+import arvados.config
from arvados._version import __version__
+logger = logging.getLogger('arvados.keepdocker')
+logger.setLevel(logging.DEBUG if arvados.config.get('ARVADOS_DEBUG')
+ else logging.INFO)
+
EARLIEST_DATETIME = datetime.datetime(datetime.MINYEAR, 1, 1, 0, 0, 0)
STAT_CACHE_ERRORS = (IOError, OSError, ValueError)
@@ -103,15 +111,15 @@ def docker_image_format(image_hash):
def docker_image_compatible(api, image_hash):
supported = api._rootDesc.get('dockerImageFormats', [])
if not supported:
- print >>sys.stderr, "arv-keepdocker: warning: server does not specify supported image formats (see docker_image_formats in server config). Continuing."
+ logger.warn("server does not specify supported image formats (see docker_image_formats in server config). Continuing.")
return True
fmt = docker_image_format(image_hash)
if fmt in supported:
return True
else:
- print >>sys.stderr, "arv-keepdocker: image format is {!r} " \
- "but server supports only {!r}".format(fmt, supported)
+ logger.error("image format is {!r} " \
+ "but server supports only {!r}".format(fmt, supported))
return False
def docker_images():
@@ -332,59 +340,82 @@ def _uuid2pdh(api, uuid):
_migration_link_class = 'docker_image_migration'
_migration_link_name = 'migrate_1.9_1.10'
-def _migrate19_link(api, root_uuid, old_uuid, new_uuid):
- old_pdh = _uuid2pdh(api, old_uuid)
- new_pdh = _uuid2pdh(api, new_uuid)
- if not api.links().list(filters=[
- ['owner_uuid', '=', root_uuid],
- ['link_class', '=', _migration_link_class],
- ['name', '=', _migration_link_name],
- ['tail_uuid', '=', old_pdh],
- ['head_uuid', '=', new_pdh]]).execute()['items']:
- print >>sys.stderr, 'Creating migration link {} -> {}: '.format(
- old_pdh, new_pdh),
- link = api.links().create(body={
- 'owner_uuid': root_uuid,
- 'link_class': _migration_link_class,
- 'name': _migration_link_name,
- 'tail_uuid': old_pdh,
- 'head_uuid': new_pdh,
- }).execute()
- print >>sys.stderr, '{}'.format(link['uuid'])
- return link
def migrate19():
- api = arvados.api('v1')
- user = api.users().current().execute()
- if not user['is_admin']:
- raise Exception("This command requires an admin token")
- root_uuid = user['uuid'][:12] + '000000000000000'
- new_image_uuids = {}
- images = list_images_in_arv(api, 2)
+ api_client = arvados.api()
+
+ images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
+
is_new = lambda img: img['dockerhash'].startswith('sha256:')
count_new = 0
+ old_images = []
for uuid, img in images:
- if not re.match(r'^[0-9a-f]{64}$', img["tag"]):
+ if img["dockerhash"].startswith("sha256:"):
continue
- key = (img["repo"], img["tag"])
- if is_new(img) and key not in new_image_uuids:
- count_new += 1
- new_image_uuids[key] = uuid
+ key = (img["repo"], img["tag"], img["timestamp"])
+ old_images.append(img)
- count_migrations = 0
- new_links = []
- for uuid, img in images:
- key = (img['repo'], img['tag'])
- if not is_new(img) and key in new_image_uuids:
- count_migrations += 1
- link = _migrate19_link(api, root_uuid, uuid, new_image_uuids[key])
- if link:
- new_links.append(link)
+ migration_links = arvados.util.list_all(api_client.links().list, filters=[
+ ['link_class', '=', _migration_link_class],
+ ['name', '=', _migration_link_name],
+ ])
+
+ already_migrated = set()
+ for m in migration_links:
+ already_migrated.add(m["tail_uuid"])
+
+ need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
+
+ logger.info("Already migrated %i images", len(already_migrated))
+ logger.info("Need to migrate %i images", len(need_migrate))
+
+ for old_image in need_migrate:
+ logger.info("Migrating %s", old_image["collection"])
+
+ col = CollectionReader(old_image["collection"])
+ tarfile = col.keys()[0]
+
+ try:
+ varlibdocker = tempfile.mkdtemp()
+ with tempfile.NamedTemporaryFile() as envfile:
+ envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
+ envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
+ envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+ envfile.flush()
+
+ dockercmd = ["docker", "run",
+ "--privileged",
+ "--rm",
+ "--env-file", envfile.name,
+ "--volume", "%s:/var/lib/docker" % varlibdocker,
+ "arvados/docker19-migrate",
+ "/root/migrate.sh",
+ "%s/%s" % (old_image["collection"], tarfile),
+ tarfile[0:40],
+ old_image["repo"],
+ old_image["tag"],
+ col.api_response()["owner_uuid"]]
+
+ out = subprocess.check_output(dockercmd)
+
+ new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
+ api_client.links().create(body={"link": {
+ 'owner_uuid': col.api_response()["owner_uuid"],
+ 'link_class': arvados.commands.keepdocker._migration_link_class,
+ 'name': arvados.commands.keepdocker._migration_link_name,
+ 'tail_uuid': old_image["collection"],
+ 'head_uuid': new_collection.group(1)
+ }}).execute(num_retries=3)
+
+ logger.info("Migrated '%s' to '%s'", old_image["collection"], new_collection.group(1))
+ except Exception as e:
+ logger.exception("Migration failed")
+ finally:
+ shutil.rmtree(varlibdocker)
+
+ logger.info("All done")
- print >>sys.stderr, "=== {} new-format images, {} migrations detected, " \
- "{} links added.".format(count_new, count_migrations, len(new_links))
- return new_links
def main(arguments=None, stdout=sys.stdout):
args = arg_parser.parse_args(arguments)
@@ -405,15 +436,15 @@ def main(arguments=None, stdout=sys.stdout):
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
- print >>sys.stderr, "arv-keepdocker:", error.message
+ logger.error(error.message)
sys.exit(1)
if not docker_image_compatible(api, image_hash):
if args.force_image_format:
- print >>sys.stderr, "arv-keepdocker: forcing incompatible image"
+ logger.warn("forcing incompatible image")
else:
- print >>sys.stderr, "arv-keepdocker: refusing to store " \
- "incompatible format (use --force-image-format to override)"
+ logger.error("refusing to store " \
+ "incompatible format (use --force-image-format to override)")
sys.exit(1)
image_repo_tag = '{}:{}'.format(args.image, args.tag) if not image_hash.startswith(args.image.lower()) else None
diff --git a/tools/docker-migrator/build.sh b/tools/docker-migrator/build.sh
deleted file mode 100755
index ecef09d..0000000
--- a/tools/docker-migrator/build.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-exec docker build -t arvados/docker19-migrate docker
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
deleted file mode 100755
index ec10b75..0000000
--- a/tools/docker-migrator/migrate.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env python
-
-import arvados
-import arvados.util
-from arvados.collection import CollectionReader
-import arvados.commands.keepdocker
-import re
-import subprocess
-import os
-import tempfile
-import shutil
-
-from pprint import pprint
-
-def main():
- api_client = arvados.api()
-
- images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
-
- is_new = lambda img: img['dockerhash'].startswith('sha256:')
-
- count_new = 0
- old_images = []
- for uuid, img in images:
- if img["dockerhash"].startswith("sha256:"):
- continue
- key = (img["repo"], img["tag"], img["timestamp"])
- old_images.append(img)
-
- migration_links = arvados.util.list_all(api_client.links().list, filters=[
- ['link_class', '=', arvados.commands.keepdocker._migration_link_class],
- ['name', '=', arvados.commands.keepdocker._migration_link_name],
- ])
-
- already_migrated = set()
- for m in migration_links:
- already_migrated.add(m["tail_uuid"])
-
- need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
-
- print "Already migrated %i images" % (len(already_migrated))
- print "Need to migrate %i images" % (len(need_migrate))
-
- for old_image in need_migrate:
- print "Migrating %s" % (old_image["collection"])
-
- col = CollectionReader(old_image["collection"])
- tarfile = col.keys()[0]
-
- try:
- varlibdocker = tempfile.mkdtemp()
- with tempfile.NamedTemporaryFile() as envfile:
- envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
- envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
- envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
- envfile.flush()
-
- dockercmd = ["docker", "run",
- "--privileged",
- "--rm",
- "--env-file", envfile.name,
- "--volume", "%s:/var/lib/docker" % varlibdocker,
- "arvados/docker19-migrate",
- "/root/migrate.sh",
- "%s/%s" % (old_image["collection"], tarfile),
- tarfile[0:40],
- old_image["repo"],
- old_image["tag"],
- col.api_response()["owner_uuid"]]
-
- out = subprocess.check_output(dockercmd)
-
- new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
- api_client.links().create(body={"link": {
- 'owner_uuid': col.api_response()["owner_uuid"],
- 'link_class': arvados.commands.keepdocker._migration_link_class,
- 'name': arvados.commands.keepdocker._migration_link_name,
- 'tail_uuid': old_image["collection"],
- 'head_uuid': new_collection.group(1)
- }}).execute(num_retries=3)
-
- print "Migrated '%s' to '%s'" % (old_image["collection"], new_collection.group(1))
- finally:
- shutil.rmtree(varlibdocker)
-
- print "All done"
-
-
-main()
commit 57fec89db11c52fb35c77792025ae0b6c61691a7
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Mar 6 14:31:17 2017 -0500
8567: Add status reporting to migrate script.
diff --git a/tools/docker-migrator/README b/tools/docker-migrator/README
new file mode 100644
index 0000000..d94eb88
--- /dev/null
+++ b/tools/docker-migrator/README
@@ -0,0 +1,24 @@
+Docker image format migration tool for Arvados.
+
+This converts Docker images stored in Arvados from image format v1
+(Docker <= 1.9) to image format v2 (Docker >= 1.10).
+
+Requires Docker running on the local host.
+
+Usage:
+
+1) Run ./build.sh to create arvados/docker19-migrate Docker image.
+
+2) Set ARVADOS_API_HOST and ARVADOS_API_TOKEN to the cluster you want to migrate.
+
+3) Run ./migrate.py
+
+This will query Arvados for v1 format Docker images. For each image that does
+not already have a corresponding v2 format image (as indicated by a
+docker_image_migration tag) it will perform the following process:
+
+i) download the image from Arvados
+ii) load it into Docker
+iii) update the Docker version, which updates the image
+iv) save the v2 format image and upload to Arvados
+v) create a migration link
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
index 23b8166..ec10b75 100755
--- a/tools/docker-migrator/migrate.py
+++ b/tools/docker-migrator/migrate.py
@@ -36,31 +36,39 @@ def main():
for m in migration_links:
already_migrated.add(m["tail_uuid"])
- for old_image in old_images:
- if old_image["collection"] in already_migrated:
- continue
+ need_migrate = [img for img in old_images if img["collection"] not in already_migrated]
+
+ print "Already migrated %i images" % (len(already_migrated))
+ print "Need to migrate %i images" % (len(need_migrate))
+
+ for old_image in need_migrate:
+ print "Migrating %s" % (old_image["collection"])
+
col = CollectionReader(old_image["collection"])
tarfile = col.keys()[0]
- varlibdocker = tempfile.mkdtemp()
-
try:
- dockercmd = ["docker", "run",
- "--privileged",
- "--rm",
- "--env", "ARVADOS_API_HOST=%s" % (os.environ["ARVADOS_API_HOST"]),
- "--env", "ARVADOS_API_TOKEN=%s" % (os.environ["ARVADOS_API_TOKEN"]),
- "--env", "ARVADOS_API_HOST_INSECURE=%s" % (os.environ["ARVADOS_API_HOST_INSECURE"]),
- "--volume", "%s:/var/lib/docker" % varlibdocker,
- "arvados/docker19-migrate",
- "/root/migrate.sh",
- "%s/%s" % (old_image["collection"], tarfile),
- tarfile[0:40],
- old_image["repo"],
- old_image["tag"],
- col.api_response()["owner_uuid"]]
-
- out = subprocess.check_output(dockercmd)
+ varlibdocker = tempfile.mkdtemp()
+ with tempfile.NamedTemporaryFile() as envfile:
+ envfile.write("ARVADOS_API_HOST=%s\n" % (os.environ["ARVADOS_API_HOST"]))
+ envfile.write("ARVADOS_API_TOKEN=%s\n" % (os.environ["ARVADOS_API_TOKEN"]))
+ envfile.write("ARVADOS_API_HOST_INSECURE=%s\n" % (os.environ["ARVADOS_API_HOST_INSECURE"]))
+ envfile.flush()
+
+ dockercmd = ["docker", "run",
+ "--privileged",
+ "--rm",
+ "--env-file", envfile.name,
+ "--volume", "%s:/var/lib/docker" % varlibdocker,
+ "arvados/docker19-migrate",
+ "/root/migrate.sh",
+ "%s/%s" % (old_image["collection"], tarfile),
+ tarfile[0:40],
+ old_image["repo"],
+ old_image["tag"],
+ col.api_response()["owner_uuid"]]
+
+ out = subprocess.check_output(dockercmd)
new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
api_client.links().create(body={"link": {
@@ -75,5 +83,7 @@ def main():
finally:
shutil.rmtree(varlibdocker)
+ print "All done"
+
main()
commit b5886da1fefdc395febeeca3a50f87337173e15d
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Mar 6 10:39:57 2017 -0500
8567: Creates migration links.
diff --git a/tools/docker-migrator/docker/migrate.sh b/tools/docker-migrator/docker/migrate.sh
index 5638950..63d7274 100755
--- a/tools/docker-migrator/docker/migrate.sh
+++ b/tools/docker-migrator/docker/migrate.sh
@@ -2,13 +2,18 @@
set -e
-#/root/dnd.sh &
-/migrator/dnd.sh &
+/root/dnd.sh &
sleep 2
-arv-get $1 | docker load
+image_tar_keepref=$1
+image_id=$2
+image_repo=$3
+image_tag=$4
+project_uuid=$5
-docker tag $2 $3:$4
+arv-get $image_tar_keepref | docker load
+
+docker tag $image_id $image_repo:$image_tag
docker images -a
@@ -18,12 +23,12 @@ sleep 1
cd /root/pkgs
dpkg -i libltdl7_2.4.2-1.11+b1_amd64.deb docker-engine_1.13.1-0~debian-jessie_amd64.deb
-/migrator/dnd.sh &
+/root/dnd.sh &
sleep 2
docker images -a
-UUID=$(arv-keepdocker --project-uuid=$5 $3 $4)
+UUID=$(arv-keepdocker --project-uuid=$project_uuid $image_repo $image_tag)
kill $(cat /var/run/docker.pid)
sleep 1
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
index 4fd7dd6..23b8166 100755
--- a/tools/docker-migrator/migrate.py
+++ b/tools/docker-migrator/migrate.py
@@ -27,8 +27,6 @@ def main():
key = (img["repo"], img["tag"], img["timestamp"])
old_images.append(img)
- print old_images
-
migration_links = arvados.util.list_all(api_client.links().list, filters=[
['link_class', '=', arvados.commands.keepdocker._migration_link_class],
['name', '=', arvados.commands.keepdocker._migration_link_name],
@@ -38,12 +36,6 @@ def main():
for m in migration_links:
already_migrated.add(m["tail_uuid"])
-# ['tail_uuid', '=', old_pdh],
-# ['head_uuid', '=', new_pdh]]).execute()['items']
-
- pprint(old_images)
- pprint(already_migrated)
-
for old_image in old_images:
if old_image["collection"] in already_migrated:
continue
@@ -66,12 +58,20 @@ def main():
tarfile[0:40],
old_image["repo"],
old_image["tag"],
- old_image["owner_uuid"]]
+ col.api_response()["owner_uuid"]]
out = subprocess.check_output(dockercmd)
new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
- print "New collection is '%s'" % new_collection.group(1)
+ api_client.links().create(body={"link": {
+ 'owner_uuid': col.api_response()["owner_uuid"],
+ 'link_class': arvados.commands.keepdocker._migration_link_class,
+ 'name': arvados.commands.keepdocker._migration_link_name,
+ 'tail_uuid': old_image["collection"],
+ 'head_uuid': new_collection.group(1)
+ }}).execute(num_retries=3)
+
+ print "Migrated '%s' to '%s'" % (old_image["collection"], new_collection.group(1))
finally:
shutil.rmtree(varlibdocker)
commit 80d9b40b6c9a8c5133aaaa4b54688027a09ba29b
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Mar 6 10:22:52 2017 -0500
8567: Docker image migration WIP.
diff --git a/tools/docker-migrator/build.sh b/tools/docker-migrator/build.sh
new file mode 100755
index 0000000..ecef09d
--- /dev/null
+++ b/tools/docker-migrator/build.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec docker build -t arvados/docker19-migrate docker
diff --git a/tools/docker-migrator/docker/Dockerfile b/tools/docker-migrator/docker/Dockerfile
new file mode 100644
index 0000000..7e921c8
--- /dev/null
+++ b/tools/docker-migrator/docker/Dockerfile
@@ -0,0 +1,31 @@
+FROM debian:8
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7 && \
+ gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+ apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D || \
+ apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+
+VOLUME /var/lib/docker
+
+RUN mkdir -p /etc/apt/sources.list.d && \
+ echo deb http://apt.arvados.org/ jessie main > /etc/apt/sources.list.d/apt.arvados.org.list && \
+ apt-get clean && \
+ apt-get update && \
+ apt-get install -yq --no-install-recommends -o Acquire::Retries=6 \
+ git curl python-arvados-python-client apt-transport-https ca-certificates && \
+ apt-get clean
+
+RUN echo deb https://apt.dockerproject.org/repo debian-jessie main > /etc/apt/sources.list.d/docker.list && \
+ apt-get update && \
+ apt-get install -yq --no-install-recommends -o Acquire::Retries=6 \
+ docker-engine=1.9.1-0~jessie && \
+ apt-get clean
+
+RUN mkdir /root/pkgs && \
+ cd /root/pkgs && \
+ curl -L -O https://apt.dockerproject.org/repo/pool/main/d/docker-engine/docker-engine_1.13.1-0~debian-jessie_amd64.deb && \
+ curl -L -O http://httpredir.debian.org/debian/pool/main/libt/libtool/libltdl7_2.4.2-1.11+b1_amd64.deb
+
+ADD migrate.sh dnd.sh /root/
diff --git a/tools/docker-migrator/docker/dnd.sh b/tools/docker-migrator/docker/dnd.sh
new file mode 100755
index 0000000..ce72601
--- /dev/null
+++ b/tools/docker-migrator/docker/dnd.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# Taken from https://github.com/jpetazzo/dind
+
+exec 2>&1
+
+# Ensure that all nodes in /dev/mapper correspond to mapped devices currently loaded by the device-mapper kernel driver
+dmsetup mknodes
+
+: {LOG:=stdio}
+
+# First, make sure that cgroups are mounted correctly.
+CGROUP=/sys/fs/cgroup
+[ -d $CGROUP ] || mkdir $CGROUP
+
+if mountpoint -q $CGROUP ; then
+ break
+else
+ mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP
+fi
+
+if ! mountpoint -q $CGROUP ; then
+ echo "Could not find or mount cgroups. Tried /sys/fs/cgroup and /cgroup. Did you use --privileged?"
+ exit 1
+fi
+
+if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security
+then
+ mount -t securityfs none /sys/kernel/security || {
+ echo "Could not mount /sys/kernel/security."
+ echo "AppArmor detection and --privileged mode might break."
+ }
+fi
+
+# Mount the cgroup hierarchies exactly as they are in the parent system.
+for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
+do
+ [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
+ mountpoint -q $CGROUP/$SUBSYS ||
+ mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
+
+ # The two following sections address a bug which manifests itself
+ # by a cryptic "lxc-start: no ns_cgroup option specified" when
+ # trying to start containers withina container.
+ # The bug seems to appear when the cgroup hierarchies are not
+ # mounted on the exact same directories in the host, and in the
+ # container.
+
+ # Named, control-less cgroups are mounted with "-o name=foo"
+ # (and appear as such under /proc/<pid>/cgroup) but are usually
+ # mounted on a directory named "foo" (without the "name=" prefix).
+ # Systemd and OpenRC (and possibly others) both create such a
+ # cgroup. To avoid the aforementioned bug, we symlink "foo" to
+ # "name=foo". This shouldn't have any adverse effect.
+ echo $SUBSYS | grep -q ^name= && {
+ NAME=$(echo $SUBSYS | sed s/^name=//)
+ ln -s $SUBSYS $CGROUP/$NAME
+ }
+
+ # Likewise, on at least one system, it has been reported that
+ # systemd would mount the CPU and CPU accounting controllers
+ # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu"
+ # but on a directory called "cpu,cpuacct" (note the inversion
+ # in the order of the groups). This tries to work around it.
+ [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct
+done
+
+# Note: as I write those lines, the LXC userland tools cannot setup
+# a "sub-container" properly if the "devices" cgroup is not in its
+# own hierarchy. Let's detect this and issue a warning.
+grep -q :devices: /proc/1/cgroup ||
+ echo "WARNING: the 'devices' cgroup should be in its own hierarchy."
+grep -qw devices /proc/1/cgroup ||
+ echo "WARNING: it looks like the 'devices' cgroup is not mounted."
+
+# Now, close extraneous file descriptors.
+pushd /proc/self/fd >/dev/null
+for FD in *
+do
+ case "$FD" in
+ # Keep stdin/stdout/stderr
+ [012])
+ ;;
+ # Nuke everything else
+ *)
+ eval exec "$FD>&-"
+ ;;
+ esac
+done
+popd >/dev/null
+
+
+# If a pidfile is still around (for example after a container restart),
+# delete it so that docker can start.
+rm -rf /var/run/docker.pid
+
+read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
+
+if ! docker daemon --storage-driver=overlay $DOCKER_DAEMON_ARGS ; then
+ docker daemon $DOCKER_DAEMON_ARGS
+fi
diff --git a/tools/docker-migrator/docker/migrate.sh b/tools/docker-migrator/docker/migrate.sh
new file mode 100755
index 0000000..5638950
--- /dev/null
+++ b/tools/docker-migrator/docker/migrate.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+#/root/dnd.sh &
+/migrator/dnd.sh &
+sleep 2
+
+arv-get $1 | docker load
+
+docker tag $2 $3:$4
+
+docker images -a
+
+kill $(cat /var/run/docker.pid)
+sleep 1
+
+cd /root/pkgs
+dpkg -i libltdl7_2.4.2-1.11+b1_amd64.deb docker-engine_1.13.1-0~debian-jessie_amd64.deb
+
+/migrator/dnd.sh &
+sleep 2
+
+docker images -a
+
+UUID=$(arv-keepdocker --project-uuid=$5 $3 $4)
+
+kill $(cat /var/run/docker.pid)
+sleep 1
+
+chmod ugo+rwx -R /var/lib/docker
+
+echo "Migrated uuid is $UUID"
diff --git a/tools/docker-migrator/migrate.py b/tools/docker-migrator/migrate.py
new file mode 100755
index 0000000..4fd7dd6
--- /dev/null
+++ b/tools/docker-migrator/migrate.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+import arvados
+import arvados.util
+from arvados.collection import CollectionReader
+import arvados.commands.keepdocker
+import re
+import subprocess
+import os
+import tempfile
+import shutil
+
+from pprint import pprint
+
+def main():
+ api_client = arvados.api()
+
+ images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3)
+
+ is_new = lambda img: img['dockerhash'].startswith('sha256:')
+
+ count_new = 0
+ old_images = []
+ for uuid, img in images:
+ if img["dockerhash"].startswith("sha256:"):
+ continue
+ key = (img["repo"], img["tag"], img["timestamp"])
+ old_images.append(img)
+
+ print old_images
+
+ migration_links = arvados.util.list_all(api_client.links().list, filters=[
+ ['link_class', '=', arvados.commands.keepdocker._migration_link_class],
+ ['name', '=', arvados.commands.keepdocker._migration_link_name],
+ ])
+
+ already_migrated = set()
+ for m in migration_links:
+ already_migrated.add(m["tail_uuid"])
+
+# ['tail_uuid', '=', old_pdh],
+# ['head_uuid', '=', new_pdh]]).execute()['items']
+
+ pprint(old_images)
+ pprint(already_migrated)
+
+ for old_image in old_images:
+ if old_image["collection"] in already_migrated:
+ continue
+ col = CollectionReader(old_image["collection"])
+ tarfile = col.keys()[0]
+
+ varlibdocker = tempfile.mkdtemp()
+
+ try:
+ dockercmd = ["docker", "run",
+ "--privileged",
+ "--rm",
+ "--env", "ARVADOS_API_HOST=%s" % (os.environ["ARVADOS_API_HOST"]),
+ "--env", "ARVADOS_API_TOKEN=%s" % (os.environ["ARVADOS_API_TOKEN"]),
+ "--env", "ARVADOS_API_HOST_INSECURE=%s" % (os.environ["ARVADOS_API_HOST_INSECURE"]),
+ "--volume", "%s:/var/lib/docker" % varlibdocker,
+ "arvados/docker19-migrate",
+ "/root/migrate.sh",
+ "%s/%s" % (old_image["collection"], tarfile),
+ tarfile[0:40],
+ old_image["repo"],
+ old_image["tag"],
+ old_image["owner_uuid"]]
+
+ out = subprocess.check_output(dockercmd)
+
+ new_collection = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
+ print "New collection is '%s'" % new_collection.group(1)
+ finally:
+ shutil.rmtree(varlibdocker)
+
+
+main()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list