[ARVADOS] created: 5acaab6b30d0a08653418dfea9702754197b0738
git at public.curoverse.com
git at public.curoverse.com
Fri Aug 8 10:46:05 EDT 2014
at 5acaab6b30d0a08653418dfea9702754197b0738 (commit)
commit 5acaab6b30d0a08653418dfea9702754197b0738
Author: Brett Smith <brett at curoverse.com>
Date: Fri Aug 8 10:46:51 2014 -0400
3527: Find Docker images natively inside Collections.
We previously relied on the docker_image_hash link to treat a
Collection as a Docker image. This was safer and simpler, but it
creates obstacles for project sharing. When we have a Collection
locator that we want to treat as a Docker image, we now check that it
has a single file with a single 64-hexdigit tar file in it, and use
that as the hash if so. This should still prevent clear mistakes
while removing some of the obstacles to sharing.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 1654646..ad43303 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1467,16 +1467,15 @@ sub must_lock_now
}
sub find_docker_hash {
- # Given a Keep locator, search for a matching link to find the Docker hash
- # of the stored image.
+ # Given a Keep locator, return the hash of the image inside it.
my $locator = shift;
- my $links_result = $arv->{links}->{list}->execute(
- filters => [["head_uuid", "=", $locator],
- ["link_class", "=", "docker_image_hash"]],
- limit => 1);
my $docker_hash;
- foreach my $link (@{$links_result->{items}}) {
- $docker_hash = lc($link->{name});
+ if (my $image = $arv->{collections}->{get}->execute(uuid => $locator)) {
+ my @file_list = @{$image->{files}};
+ if ((scalar(@file_list) == 1) &&
+ ($file_list[0][1] =~ /^([0-9A-Fa-f]{64})\.tar$/)) {
+ $docker_hash = $1;
+ }
}
return $docker_hash;
}
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index a50c47b..2601a27 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -159,12 +159,17 @@ class Collection < ArvadosModel
joins("JOIN collections ON links.head_uuid = collections.uuid").
order("links.created_at DESC")
- # If the search term is a Collection locator with an associated
- # Docker image hash link, return that Collection.
- coll_matches = base_search.
- where(link_class: "docker_image_hash", collections: {uuid: search_term})
- if match = coll_matches.first
- return [match.head_uuid]
+ # If the search term is a Collection locator that contains one file
+ # that looks like a Docker image, return it.
+ if loc = Locator.parse(search_term)
+ loc.strip_hints!
+ coll_match = readable_by(*readers).where(uuid: loc.to_s).first
+ if coll_match.andand.files.andand.size == 1
+ dirname, basename = coll_match.files.first[0, 2]
+ if (dirname == ".") and (basename =~ /^[0-9A-Fa-f]{64}\.tar$/)
+ return [loc.to_s]
+ end
+ end
end
# Find Collections with matching Docker image repository+tag pairs.
diff --git a/services/api/app/models/locator.rb b/services/api/app/models/locator.rb
index 39d7da9..d01c9dc 100644
--- a/services/api/app/models/locator.rb
+++ b/services/api/app/models/locator.rb
@@ -11,7 +11,7 @@
# hint-content ::= [A-Za-z0-9 at _-]+
#
# Individual hints may have their own required format:
-#
+#
# sign-hint ::= "+A" <40 lowercase hex digits> "@" sign-timestamp
# sign-timestamp ::= <8 lowercase hex digits>
@@ -66,6 +66,15 @@ class Locator
Locator.new(@hash, @size, @hints.reject { |o| o.start_with?("A") })
end
+ def strip_hints
+ Locator.new(@hash, @size, [])
+ end
+
+ def strip_hints!
+ @hints = []
+ self
+ end
+
def hash
@hash
end
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 8a11803..fbd150c 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -70,6 +70,18 @@ docker_image:
updated_at: 2014-06-11T17:22:54Z
manifest_text: ". d21353cfe035e3e384563ee55eadbb2f+67108864 5c77a43e329b9838cbec18ff42790e57+55605760 0:122714624:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678.tar\n"
+unlinked_docker_image:
+ # This Collection contains a file that looks like a Docker image,
+ # but has no Docker metadata links pointing to it.
+ uuid: 9ae44d5792468c58bcf85ce7353c7027+124
+ owner_uuid: qr1hi-tpzed-000000000000000
+ created_at: 2014-06-11T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-06-11T17:22:54Z
+ updated_at: 2014-06-11T17:22:54Z
+ manifest_text: ". fca529cfe035e3e384563ee55eadbb2f+67108863 0:67108863:bcd02158b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678.tar\n"
+
empty:
# Empty collection owned by anonymous_group is added with rake db:seed.
uuid: d41d8cd98f00b204e9800998ecf8427e+0
diff --git a/services/api/test/fixtures/links.yml b/services/api/test/fixtures/links.yml
index ba0b1a6..1afafaf 100644
--- a/services/api/test/fixtures/links.yml
+++ b/services/api/test/fixtures/links.yml
@@ -539,6 +539,20 @@ active_user_permission_to_docker_image_collection:
head_uuid: fa3c1a9cb6783f85f2ecda037e07b8c3+167
properties: {}
+active_user_permission_to_unlinked_docker_image_collection:
+ uuid: zzzzz-o0j2j-g5i0sa8cr3b1psf
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_read
+ head_uuid: 9ae44d5792468c58bcf85ce7353c7027+124
+ properties: {}
+
docker_image_collection_hash:
uuid: zzzzz-o0j2j-dockercollhasha
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
diff --git a/services/api/test/unit/job_test.rb b/services/api/test/unit/job_test.rb
index e1ca7c5..73a676f 100644
--- a/services/api/test/unit/job_test.rb
+++ b/services/api/test/unit/job_test.rb
@@ -103,6 +103,13 @@ class JobTest < ActiveSupport::TestCase
assert(job.invalid?, "non-Docker Collection constraint was valid")
end
+ test "can create Job with Docker image Collection without Docker links" do
+ image_uuid = collections(:unlinked_docker_image).uuid
+ job = Job.new job_attrs(runtime_constraints: {"docker_image" => image_uuid})
+ assert(job.valid?, "Job created with unlinked Docker image was invalid")
+ assert_equal(image_uuid, job.docker_image_locator)
+ end
+
test "can't create Job with Docker image locator" do
begin
job = Job.new job_attrs(docker_image_locator: BAD_COLLECTION)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list