[ARVADOS] created: 7c667d8963c7a3cf9acd04c1d938b5273b761228
Git user
git at public.curoverse.com
Mon Feb 13 17:18:38 EST 2017
at 7c667d8963c7a3cf9acd04c1d938b5273b761228 (commit)
commit 7c667d8963c7a3cf9acd04c1d938b5273b761228
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Feb 13 17:18:11 2017 -0500
11017: Add arv-migrate-docker19
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 5f7749b..3ffb7f3 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -6,6 +6,7 @@ import datetime
import errno
import json
import os
+import re
import subprocess
import sys
import tarfile
@@ -323,6 +324,68 @@ def list_images_in_arv(api_client, num_retries, image_name=None, image_tag=None)
def items_owned_by(owner_uuid, arv_items):
return (item for item in arv_items if item['owner_uuid'] == owner_uuid)
+def _uuid2pdh(api, uuid):
+ return api.collections().list(
+ filters=[['uuid', '=', uuid]],
+ select=['portable_data_hash'],
+ ).execute()['items'][0]['portable_data_hash']
+
+_migration_link_class = 'docker_image_migration'
+_migration_link_name = 'migrate_1.9_1.10'
+def _migrate19_link(api, root_uuid, old_uuid, new_uuid):
+ old_pdh = _uuid2pdh(api, old_uuid)
+ new_pdh = _uuid2pdh(api, new_uuid)
+ if not api.links().list(filters=[
+ ['owner_uuid', '=', root_uuid],
+ ['link_class', '=', _migration_link_class],
+ ['name', '=', _migration_link_name],
+ ['tail_uuid', '=', old_pdh],
+ ['head_uuid', '=', new_pdh]]).execute()['items']:
+ print >>sys.stderr, 'Creating migration link {} -> {}: '.format(
+ old_pdh, new_pdh),
+ link = api.links().create(body={
+ 'owner_uuid': root_uuid,
+ 'link_class': _migration_link_class,
+ 'name': _migration_link_name,
+ 'tail_uuid': old_pdh,
+ 'head_uuid': new_pdh,
+ }).execute()
+ print >>sys.stderr, '{}'.format(link['uuid'])
+ return link
+
+def migrate19():
+ api = arvados.api('v1')
+ user = api.users().current().execute()
+ if not user['is_admin']:
+ raise Exception("This command requires an admin token")
+ root_uuid = user['uuid'][:12] + '000000000000000'
+ new_image_uuids = {}
+ images = list_images_in_arv(api, 2)
+ is_new = lambda img: img['dockerhash'].startswith('sha256:')
+
+ count_new = 0
+ for uuid, img in images:
+ if not re.match(r'^[0-9a-f]{64}$', img["tag"]):
+ continue
+ key = (img["repo"], img["tag"])
+ if is_new(img) and key not in new_image_uuids:
+ count_new += 1
+ new_image_uuids[key] = uuid
+
+ count_migrations = 0
+ new_links = []
+ for uuid, img in images:
+ key = (img['repo'], img['tag'])
+ if not is_new(img) and key in new_image_uuids:
+ count_migrations += 1
+ link = _migrate19_link(api, root_uuid, uuid, new_image_uuids[key])
+ if link:
+ new_links.append(link)
+
+ print >>sys.stderr, "=== {} new-format images, {} migrations detected, " \
+ "{} links added.".format(count_new, count_migrations, len(new_links))
+ return new_links
+
def main(arguments=None, stdout=sys.stdout):
args = arg_parser.parse_args(arguments)
api = arvados.api('v1')
diff --git a/sdk/python/bin/arv-migrate-docker19 b/sdk/python/bin/arv-migrate-docker19
new file mode 100755
index 0000000..fcef8d7
--- /dev/null
+++ b/sdk/python/bin/arv-migrate-docker19
@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+
+from arvados.commands.keepdocker import migrate19
+migrate19()
diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py
index 151edf3..9fcf2f1 100644
--- a/sdk/python/tests/test_arv_keepdocker.py
+++ b/sdk/python/tests/test_arv_keepdocker.py
@@ -13,6 +13,7 @@ import unittest
import arvados.commands.keepdocker as arv_keepdocker
import arvados_testutil as tutil
+import run_test_server
class StopTest(Exception):
@@ -37,6 +38,22 @@ class ArvKeepdockerTestCase(unittest.TestCase):
self.assertEqual(out.getvalue(), '')
self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ def test_migrate19(self):
+ try:
+ sys.argv = ['arv-migrate-docker19']
+
+ added = arv_keepdocker.migrate19()
+ self.assertEqual(len(added), 1)
+ self.assertEqual(added[0]['link_class'], 'docker_image_migration')
+ self.assertEqual(added[0]['name'], 'migrate_1.9_1.10')
+ self.assertEqual(added[0]['tail_uuid'], 'fa3c1a9cb6783f85f2ecda037e07b8c3+167')
+ self.assertEqual(added[0]['head_uuid'], 'd740a57097711e08eb9b2a93518f20ab+174')
+
+ added = arv_keepdocker.migrate19()
+ self.assertEqual(added, [])
+ finally:
+ run_test_server.reset()
+
@mock.patch('arvados.commands.keepdocker.find_image_hashes',
return_value=['abc123'])
@mock.patch('arvados.commands.keepdocker.find_one_image_hash',
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 2eb873b..cda7515 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -108,7 +108,7 @@ docker_image:
manifest_text: ". d21353cfe035e3e384563ee55eadbb2f+67108864 5c77a43e329b9838cbec18ff42790e57+55605760 0:122714624:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678.tar\n"
name: docker_image
-# untagged docker image with sha256:{hash}.tar filename
+# tagged docker image with sha256:{hash}.tar filename
docker_image_1_12:
uuid: zzzzz-4zz18-1g4g0vhpjn9wq7i
portable_data_hash: d740a57097711e08eb9b2a93518f20ab+174
diff --git a/services/api/test/fixtures/links.yml b/services/api/test/fixtures/links.yml
index 7d9aea5..ac579f7 100644
--- a/services/api/test/fixtures/links.yml
+++ b/services/api/test/fixtures/links.yml
@@ -638,6 +638,62 @@ docker_image_collection_tag2:
properties:
image_timestamp: "2014-06-10T14:30:00.184019565Z"
+docker_image_collection_hextag:
+ uuid: zzzzz-o0j2j-2591ao7zubhaoxh
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2017-02-13 21:41:06.769936997 Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2017-02-13 21:41:06.769422000 Z
+ tail_uuid: ~
+ link_class: docker_image_repo+tag
+ name: arvados/apitestfixture:b25678748af0cac6d1180b9ca4ce3ef31f2b06602f471aad8dfd421e149b0d75
+ head_uuid: zzzzz-4zz18-1v45jub259sjjgb
+ properties: {}
+ updated_at: 2017-02-13 21:41:06.769422000 Z
+
+docker_1_12_image_hash:
+ uuid: zzzzz-o0j2j-f58l58fn65n8v6k
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2017-02-13 21:35:12.602828136 Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2017-02-13 21:35:12.602309000 Z
+ tail_uuid: ~
+ link_class: docker_image_hash
+ name: sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678
+ head_uuid: zzzzz-4zz18-1g4g0vhpjn9wq7i
+ properties: {}
+ updated_at: 2017-02-13 21:35:12.602309000 Z
+
+docker_1_12_image_tag:
+ uuid: zzzzz-o0j2j-dybsy0m3u96jkbv
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2017-02-13 21:37:47.441406362 Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2017-02-13 21:37:47.440882000 Z
+ tail_uuid: ~
+ link_class: docker_image_repo+tag
+ name: arvados/apitestfixture:latest
+ head_uuid: zzzzz-4zz18-1g4g0vhpjn9wq7i
+ properties: {}
+ updated_at: 2017-02-13 21:37:47.440882000 Z
+
+docker_1_12_image_hextag:
+ uuid: zzzzz-o0j2j-06hzef4u1hbk1g5
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2017-02-13 21:37:47.441406362 Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2017-02-13 21:37:47.440882000 Z
+ tail_uuid: ~
+ link_class: docker_image_repo+tag
+ name: arvados/apitestfixture:b25678748af0cac6d1180b9ca4ce3ef31f2b06602f471aad8dfd421e149b0d75
+ head_uuid: zzzzz-4zz18-1g4g0vhpjn9wq7i
+ properties: {}
+ updated_at: 2017-02-13 21:37:47.440882000 Z
+
ancient_docker_image_collection_hash:
# This image helps test that searches for Docker images find
# the latest available image: the hash is the same as
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list