[ARVADOS] updated: eef298a7cf58771583b255fb0e3e1cfcbcdb6e48
git at public.curoverse.com
git at public.curoverse.com
Fri Aug 8 13:52:11 EDT 2014
Summary of changes:
sdk/cli/bin/crunch-job | 29 +++++++++++++++--------------
services/api/app/models/collection.rb | 15 +++++++++------
services/api/app/models/locator.rb | 11 ++++++++++-
services/api/test/fixtures/collections.yml | 12 ++++++++++++
services/api/test/fixtures/links.yml | 14 ++++++++++++++
services/api/test/unit/job_test.rb | 7 +++++++
6 files changed, 67 insertions(+), 21 deletions(-)
via eef298a7cf58771583b255fb0e3e1cfcbcdb6e48 (commit)
via 2945ea907a7c35da2d03d6775574fb1ad9be2b09 (commit)
via e932523a818a5d305ae541379b531a381f06edf4 (commit)
from d36a4308d2e3dd564ddee92347466833c57ccfc9 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit eef298a7cf58771583b255fb0e3e1cfcbcdb6e48
Merge: d36a430 2945ea9
Author: Brett Smith <brett at curoverse.com>
Date: Fri Aug 8 13:51:25 2014 -0400
Merge branch '3527-infer-docker-hash-wip'
Closes #3527, #3529.
commit 2945ea907a7c35da2d03d6775574fb1ad9be2b09
Author: Brett Smith <brett at curoverse.com>
Date: Fri Aug 8 13:44:00 2014 -0400
3527: Support using a Docker image from any stream in a Collection.
This makes Docker image detection more consistent between crunch-job
and the API server, and is more user-friendly.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index ad43303..06b3da9 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -499,16 +499,17 @@ if (!$have_slurm)
# If this job requires a Docker image, install that.
my $docker_bin = "/usr/bin/docker.io";
-my ($docker_locator, $docker_hash);
+my ($docker_locator, $docker_stream, $docker_hash);
if ($docker_locator = $Job->{docker_image_locator}) {
- $docker_hash = find_docker_hash($docker_locator);
+ ($docker_stream, $docker_hash) = find_docker_image($docker_locator);
if (!$docker_hash)
{
croak("No Docker image hash found from locator $docker_locator");
}
+ $docker_stream =~ s/^\.//;
my $docker_install_script = qq{
if ! $docker_bin images -q --no-trunc | grep -qxF \Q$docker_hash\E; then
- arv-get \Q$docker_locator/$docker_hash.tar\E | $docker_bin load
+ arv-get \Q$docker_locator$docker_stream/$docker_hash.tar\E | $docker_bin load
fi
};
my $docker_pid = fork();
@@ -1466,18 +1467,19 @@ sub must_lock_now
}
}
-sub find_docker_hash {
- # Given a Keep locator, return the hash of the image inside it.
+sub find_docker_image {
+ # Given a Keep locator, check to see if it contains a Docker image.
+ # If so, return its stream name and Docker hash.
+ # If not, return undef for both values.
my $locator = shift;
- my $docker_hash;
if (my $image = $arv->{collections}->{get}->execute(uuid => $locator)) {
my @file_list = @{$image->{files}};
if ((scalar(@file_list) == 1) &&
($file_list[0][1] =~ /^([0-9A-Fa-f]{64})\.tar$/)) {
- $docker_hash = $1;
+ return ($file_list[0][0], $1);
}
}
- return $docker_hash;
+ return (undef, undef);
}
__DATA__
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 2601a27..50dd42c 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -164,11 +164,9 @@ class Collection < ArvadosModel
if loc = Locator.parse(search_term)
loc.strip_hints!
coll_match = readable_by(*readers).where(uuid: loc.to_s).first
- if coll_match.andand.files.andand.size == 1
- dirname, basename = coll_match.files.first[0, 2]
- if (dirname == ".") and (basename =~ /^[0-9A-Fa-f]{64}\.tar$/)
- return [loc.to_s]
- end
+ if coll_match and (coll_match.files.size == 1) and
+ (coll_match.files[0][1] =~ /^[0-9A-Fa-f]{64}\.tar$/)
+ return [loc.to_s]
end
end
commit e932523a818a5d305ae541379b531a381f06edf4
Author: Brett Smith <brett at curoverse.com>
Date: Fri Aug 8 10:46:51 2014 -0400
3527: Find Docker images natively inside Collections.
We previously relied on the docker_image_hash link to treat a
Collection as a Docker image. This was safer and simpler, but it
creates obstacles for project sharing. When we have a Collection
locator that we want to treat as a Docker image, we now check that it
has a single file with a single 64-hexdigit tar file in it, and use
that as the hash if so. This should still prevent clear mistakes
while removing some of the obstacles to sharing.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 1654646..ad43303 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1467,16 +1467,15 @@ sub must_lock_now
}
sub find_docker_hash {
- # Given a Keep locator, search for a matching link to find the Docker hash
- # of the stored image.
+ # Given a Keep locator, return the hash of the image inside it.
my $locator = shift;
- my $links_result = $arv->{links}->{list}->execute(
- filters => [["head_uuid", "=", $locator],
- ["link_class", "=", "docker_image_hash"]],
- limit => 1);
my $docker_hash;
- foreach my $link (@{$links_result->{items}}) {
- $docker_hash = lc($link->{name});
+ if (my $image = $arv->{collections}->{get}->execute(uuid => $locator)) {
+ my @file_list = @{$image->{files}};
+ if ((scalar(@file_list) == 1) &&
+ ($file_list[0][1] =~ /^([0-9A-Fa-f]{64})\.tar$/)) {
+ $docker_hash = $1;
+ }
}
return $docker_hash;
}
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index a50c47b..2601a27 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -159,12 +159,17 @@ class Collection < ArvadosModel
joins("JOIN collections ON links.head_uuid = collections.uuid").
order("links.created_at DESC")
- # If the search term is a Collection locator with an associated
- # Docker image hash link, return that Collection.
- coll_matches = base_search.
- where(link_class: "docker_image_hash", collections: {uuid: search_term})
- if match = coll_matches.first
- return [match.head_uuid]
+ # If the search term is a Collection locator that contains one file
+ # that looks like a Docker image, return it.
+ if loc = Locator.parse(search_term)
+ loc.strip_hints!
+ coll_match = readable_by(*readers).where(uuid: loc.to_s).first
+ if coll_match.andand.files.andand.size == 1
+ dirname, basename = coll_match.files.first[0, 2]
+ if (dirname == ".") and (basename =~ /^[0-9A-Fa-f]{64}\.tar$/)
+ return [loc.to_s]
+ end
+ end
end
# Find Collections with matching Docker image repository+tag pairs.
diff --git a/services/api/app/models/locator.rb b/services/api/app/models/locator.rb
index 39d7da9..d01c9dc 100644
--- a/services/api/app/models/locator.rb
+++ b/services/api/app/models/locator.rb
@@ -11,7 +11,7 @@
# hint-content ::= [A-Za-z0-9 at _-]+
#
# Individual hints may have their own required format:
-#
+#
# sign-hint ::= "+A" <40 lowercase hex digits> "@" sign-timestamp
# sign-timestamp ::= <8 lowercase hex digits>
@@ -66,6 +66,15 @@ class Locator
Locator.new(@hash, @size, @hints.reject { |o| o.start_with?("A") })
end
+ def strip_hints
+ Locator.new(@hash, @size, [])
+ end
+
+ def strip_hints!
+ @hints = []
+ self
+ end
+
def hash
@hash
end
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 8a11803..fbd150c 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -70,6 +70,18 @@ docker_image:
updated_at: 2014-06-11T17:22:54Z
manifest_text: ". d21353cfe035e3e384563ee55eadbb2f+67108864 5c77a43e329b9838cbec18ff42790e57+55605760 0:122714624:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678.tar\n"
+unlinked_docker_image:
+ # This Collection contains a file that looks like a Docker image,
+ # but has no Docker metadata links pointing to it.
+ uuid: 9ae44d5792468c58bcf85ce7353c7027+124
+ owner_uuid: qr1hi-tpzed-000000000000000
+ created_at: 2014-06-11T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-06-11T17:22:54Z
+ updated_at: 2014-06-11T17:22:54Z
+ manifest_text: ". fca529cfe035e3e384563ee55eadbb2f+67108863 0:67108863:bcd02158b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678.tar\n"
+
empty:
# Empty collection owned by anonymous_group is added with rake db:seed.
uuid: d41d8cd98f00b204e9800998ecf8427e+0
diff --git a/services/api/test/fixtures/links.yml b/services/api/test/fixtures/links.yml
index 5e95480..4125757 100644
--- a/services/api/test/fixtures/links.yml
+++ b/services/api/test/fixtures/links.yml
@@ -539,6 +539,20 @@ active_user_permission_to_docker_image_collection:
head_uuid: fa3c1a9cb6783f85f2ecda037e07b8c3+167
properties: {}
+active_user_permission_to_unlinked_docker_image_collection:
+ uuid: zzzzz-o0j2j-g5i0sa8cr3b1psf
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_read
+ head_uuid: 9ae44d5792468c58bcf85ce7353c7027+124
+ properties: {}
+
docker_image_collection_hash:
uuid: zzzzz-o0j2j-dockercollhasha
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
diff --git a/services/api/test/unit/job_test.rb b/services/api/test/unit/job_test.rb
index e1ca7c5..73a676f 100644
--- a/services/api/test/unit/job_test.rb
+++ b/services/api/test/unit/job_test.rb
@@ -103,6 +103,13 @@ class JobTest < ActiveSupport::TestCase
assert(job.invalid?, "non-Docker Collection constraint was valid")
end
+ test "can create Job with Docker image Collection without Docker links" do
+ image_uuid = collections(:unlinked_docker_image).uuid
+ job = Job.new job_attrs(runtime_constraints: {"docker_image" => image_uuid})
+ assert(job.valid?, "Job created with unlinked Docker image was invalid")
+ assert_equal(image_uuid, job.docker_image_locator)
+ end
+
test "can't create Job with Docker image locator" do
begin
job = Job.new job_attrs(docker_image_locator: BAD_COLLECTION)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list