[ARVADOS] created: 433c48edb1523809bb8d43e4fc40ebe1a177e103
Git user
git at public.curoverse.com
Thu Mar 30 11:15:22 EDT 2017
at 433c48edb1523809bb8d43e4fc40ebe1a177e103 (commit)
commit 433c48edb1523809bb8d43e4fc40ebe1a177e103
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Mar 30 10:07:52 2017 -0400
8567: Improve disk space accounting. Default to use overlay storage driver (but user can override).
diff --git a/docker/migrate-docker19/build.sh b/docker/migrate-docker19/build.sh
index 3a36cd4..e563bfc 100755
--- a/docker/migrate-docker19/build.sh
+++ b/docker/migrate-docker19/build.sh
@@ -1,2 +1,2 @@
#!/bin/sh
-exec docker build -t arvados/migrate-docker19 .
+exec docker build -t arvados/migrate-docker19:1.0 .
diff --git a/docker/migrate-docker19/dnd.sh b/docker/migrate-docker19/dnd.sh
index ec6f1e3..f253f0b 100755
--- a/docker/migrate-docker19/dnd.sh
+++ b/docker/migrate-docker19/dnd.sh
@@ -96,4 +96,4 @@ rm -rf /var/run/docker.pid
read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
-exec docker daemon --storage-driver=vfs $DOCKER_DAEMON_ARGS
+exec docker daemon --storage-driver=$1 $DOCKER_DAEMON_ARGS
diff --git a/docker/migrate-docker19/migrate.sh b/docker/migrate-docker19/migrate.sh
index 8e39be4..350ae4c 100755
--- a/docker/migrate-docker19/migrate.sh
+++ b/docker/migrate-docker19/migrate.sh
@@ -2,8 +2,19 @@
set -e
+image_tar_keepref=$1
+image_id=$2
+image_repo=$3
+image_tag=$4
+project_uuid=$5
+graph_driver=$6
+
+function freespace() {
+ df -B1 /var/lib/docker | tail -n1 | sed 's/ */ /g' | cut -d' ' -f4
+}
+
function start_docker {
- /root/dnd.sh &
+ /root/dnd.sh $graph_driver &
for i in $(seq 1 10) ; do
if docker version >/dev/null 2>/dev/null ; then
return
@@ -27,20 +38,18 @@ function kill_docker {
}
function cleanup {
+ trap EXIT
kill_docker
rm -rf /var/lib/docker/*
rm -rf /root/.cache/arvados/docker/*
+ echo "Available space after cleanup is $(freespace)"
}
trap cleanup EXIT
start_docker
-image_tar_keepref=$1
-image_id=$2
-image_repo=$3
-image_tag=$4
-project_uuid=$5
+echo "Initial available space is $(freespace)"
arv-get $image_tar_keepref | docker load
@@ -50,13 +59,19 @@ docker images -a
kill_docker
+echo "Available space after image load is $(freespace)"
+
cd /root/pkgs
dpkg -i libltdl7_2.4.2-1.11+b1_amd64.deb docker-engine_1.13.1-0~debian-jessie_amd64.deb
+echo "Available space after image upgrade is $(freespace)"
+
start_docker
docker images -a
UUID=$(arv-keepdocker --force-image-format --project-uuid=$project_uuid $image_repo $image_tag)
+echo "Available space after arv-keepdocker is $(freespace)"
+
echo "Migrated uuid is $UUID"
diff --git a/sdk/python/arvados/commands/migrate19.py b/sdk/python/arvados/commands/migrate19.py
index 4be9996..a9768a0 100644
--- a/sdk/python/arvados/commands/migrate19.py
+++ b/sdk/python/arvados/commands/migrate19.py
@@ -56,6 +56,14 @@ def main(arguments=None):
migrate19_parser.add_argument(
'--version', action='version', version="%s %s" % (sys.argv[0], __version__),
help='Print version and exit.')
+ migrate19_parser.add_argument(
+ '--verbose', action="store_true", help="Print stdout/stderr even on success")
+ migrate19_parser.add_argument(
+ '--force', action="store_true", help="Try to migrate even if there isn't enough space")
+
+ migrate19_parser.add_argument(
+ '--storage-driver', type=str, default="overlay",
+ help="Docker storage driver, e.g. aufs, overlay, vfs")
exgroup = migrate19_parser.add_mutually_exclusive_group()
exgroup.add_argument(
@@ -74,6 +82,9 @@ def main(arguments=None):
if args.tempdir:
tempfile.tempdir = args.tempdir
+ if args.verbose:
+ logger.setLevel(logging.DEBUG)
+
only_migrate = None
if args.infile:
only_migrate = set()
@@ -115,6 +126,7 @@ def main(arguments=None):
need_migrate = {}
biggest = 0
+ biggest_pdh = None
for img in old_images:
i = uuid_to_collection[img["collection"]]
pdh = i["portable_data_hash"]
@@ -123,17 +135,34 @@ def main(arguments=None):
with CollectionReader(i["manifest_text"]) as c:
if c.values()[0].size() > biggest:
biggest = c.values()[0].size()
+ biggest_pdh = pdh
+
+
+ if args.storage_driver == "vfs":
+ will_need = (biggest*20)
+ else:
+ will_need = (biggest*2.5)
if args.print_unmigrated:
only_migrate = set()
for pdh in need_migrate:
- print pdh
+ print(pdh)
return
logger.info("Already migrated %i images", len(already_migrated))
logger.info("Need to migrate %i images", len(need_migrate))
logger.info("Using tempdir %s", tempfile.gettempdir())
- logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", biggest/(2**20), (biggest*2)/(2**20))
+ logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest/(2**20))
+
+ df_out = subprocess.check_output(["df", "-B1", tempfile.gettempdir()])
+ ln = df_out.splitlines()[1]
+ filesystem, blocks, used, available, use_pct, mounted = re.match(r"^([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+)", ln).groups(1)
+ if int(available) <= will_need:
+ logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)/(2**20), will_need/(2**20))
+ if not args.force:
+ exit(1)
+ else:
+ logger.warn("--force provided, will migrate anyway")
if args.dry_run:
return
@@ -169,23 +198,51 @@ def main(arguments=None):
"--env-file", envfile.name,
"--volume", "%s:/var/lib/docker" % varlibdocker,
"--volume", "%s:/root/.cache/arvados/docker" % dockercache,
- "arvados/migrate-docker19",
+ "arvados/migrate-docker19:1.0",
"/root/migrate.sh",
"%s/%s" % (old_image["collection"], tarfile),
tarfile[0:40],
old_image["repo"],
old_image["tag"],
- uuid_to_collection[old_image["collection"]]["owner_uuid"]]
+ uuid_to_collection[old_image["collection"]]["owner_uuid"],
+ args.storage_driver]
proc = subprocess.Popen(dockercmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = proc.communicate()
+ initial_space = re.search(r"Initial available space is (\d+)", out)
+ imgload_space = re.search(r"Available space after image load is (\d+)", out)
+ imgupgrade_space = re.search(r"Available space after image upgrade is (\d+)", out)
+ keepdocker_space = re.search(r"Available space after arv-keepdocker is (\d+)", out)
+ cleanup_space = re.search(r"Available space after cleanup is (\d+)", out)
+
+ if initial_space:
+ isp = int(initial_space.group(1))
+ logger.debug("Available space initially: %i MiB", (isp)/(2**20))
+ if imgload_space:
+ sp = int(imgload_space.group(1))
+ logger.debug("Used after load: %i MiB", (isp-sp)/(2**20))
+ if imgupgrade_space:
+ sp = int(imgupgrade_space.group(1))
+ logger.debug("Used after upgrade: %i MiB", (isp-sp)/(2**20))
+ if keepdocker_space:
+ sp = int(keepdocker_space.group(1))
+ logger.debug("Used after upload: %i MiB", (isp-sp)/(2**20))
+
+ if cleanup_space:
+ sp = int(cleanup_space.group(1))
+ logger.info("Available after cleanup: %i MiB", (sp)/(2**20))
+
if proc.returncode != 0:
logger.error("Failed with return code %i", proc.returncode)
logger.error("--- Stdout ---\n%s", out)
logger.error("--- Stderr ---\n%s", err)
raise MigrationFailed()
+ if args.verbose:
+ logger.info("--- Stdout ---\n%s", out)
+ logger.info("--- Stderr ---\n%s", err)
+
migrated = re.search(r"Migrated uuid is ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15})", out)
if migrated:
newcol = CollectionReader(migrated.group(1))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list