[ARVADOS] updated: 17c73de66fe0fa797407e824e62d3b8159b8be4b
git at public.curoverse.com
git at public.curoverse.com
Fri Jun 13 15:19:54 EDT 2014
Summary of changes:
sdk/python/arvados/commands/keepdocker.py | 3 +-
services/api/app/models/collection.rb | 49 +++++++++++++++++++++++++++++++
services/api/app/models/job.rb | 39 ++++--------------------
services/api/test/fixtures/links.yml | 8 ++---
services/api/test/unit/job_test.rb | 11 +++++--
5 files changed, 70 insertions(+), 40 deletions(-)
via 17c73de66fe0fa797407e824e62d3b8159b8be4b (commit)
via b64bddbe99166b02f979d7f47382308d31df7eee (commit)
via 4c787bace891a9e1290fd64e75428929d8049989 (commit)
from 88dc02bbde0cb0d6b916a98e4bb789412420d9ad (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 17c73de66fe0fa797407e824e62d3b8159b8be4b
Author: Brett Smith <brett at curoverse.com>
Date: Fri Jun 13 15:20:42 2014 -0400
2879: Refactor Job Docker image locating.
Having this as a Collections class method will let us use this in the
Job reuse logic. I'm hoping the implementation changes to search help
improve readability too.
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 745f0bf..bd2fb44 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -147,4 +147,53 @@ class Collection < ArvadosModel
raise "uuid #{uuid} has no hash part" if !hash_part
[hash_part, size_part].compact.join '+'
end
+
+ def self.for_latest_docker_image(search_term, search_tag=nil)
+ base_search = self.joins(:links_via_head).order("links.created_at DESC")
+ # If the search term is a Collection locator with an associated
+ # Docker image hash link, return that Collection.
+ coll_matches = base_search.
+ where(uuid: search_term, links: {link_class: 'docker_image_hash'})
+ if coll = coll_matches.first
+ return coll
+ end
+
+ # Find Collections with matching Docker image repository+tag pairs.
+ repo_matches = base_search.
+ where(links: {link_class: 'docker_image_repo+tag',
+ name: "#{search_term}:#{search_tag || 'latest'}"})
+
+ # Find Collections with matching Docker image hashes, unless we're
+ # obviously doing a repo+tag search and already found a match that way.
+ if search_tag.nil? or repo_matches.first.nil?
+ hash_matches = base_search.
+ where("links.link_class = ? and links.name LIKE ?",
+ "docker_image_hash", "#{search_term}%")
+ else
+ hash_matches = nil
+ end
+
+ # Select the image that was created most recently from both repo
+ # and hash matches.
+ latest_image = nil
+ latest_image_timestamp = "1900-01-01T00:00:00Z"
+ [repo_matches, hash_matches].compact.each do |search_result|
+ search_result.find_each do |coll|
+ coll_latest_timestamp = "1900-01-01T00:00:00Z"
+
+ coll.links_via_head.find_each do |link|
+ this_link_timestamp = link.properties["image_timestamp"]
+ if (this_link_timestamp.andand > coll_latest_timestamp)
+ coll_latest_timestamp = this_link_timestamp
+ end
+ end
+
+ if coll_latest_timestamp > latest_image_timestamp
+ latest_image = coll
+ latest_image_timestamp = coll_latest_timestamp
+ end
+ end
+ end
+ latest_image
+ end
end
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index 3073b07..db0734f 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -104,40 +104,13 @@ class Job < ArvadosModel
def find_docker_image_locator
# Find the Collection that holds the Docker image specified in the
# runtime constraints, and store its locator in docker_image_locator.
- image_name = runtime_constraints['docker_image']
- if image_name.nil?
+ image_search = runtime_constraints['docker_image']
+ image_tag = runtime_constraints['docker_image_tag']
+ if image_search.nil?
self.docker_image_locator = nil
- return
- elsif not (new_record? or runtime_constraints_changed?)
- return
- # Accept images specified as a locator, if there's an associated
- # docker_image_hash link.
- elsif coll = Collection.joins(:links_via_head).
- where(uuid: image_name,
- links: {link_class: 'docker_image_hash'}).first
- self.docker_image_locator = (coll.links_via_head.any?) ? coll.uuid : nil
- else # Accept images specified by their Docker metadata.
- if image_name =~ /^[0-9A-Fa-f]{64}$/
- link_search = {
- link_class: 'docker_image_hash',
- name: image_name,
- }
- else
- link_search = {
- link_class: 'docker_image_repo+tag',
- name: "#{image_name}:#{runtime_constraints['docker_image_tag'] || 'latest'}",
- }
- end
- links = Link.where(link_search)
- # Select the image that was created most recently.
- latest = links.first
- links.find_each do |link|
- latest = link if (link.properties['image_timestamp'] >
- latest.properties['image_timestamp'])
- end
- self.docker_image_locator = latest.andand.head_uuid
- end
- if docker_image_locator.nil?
+ elsif coll = Collection.for_latest_docker_image(image_search, image_tag)
+ self.docker_image_locator = coll.uuid
+ else
errors.add(:docker_image_locator, "Docker image not found")
false
end
diff --git a/services/api/test/unit/job_test.rb b/services/api/test/unit/job_test.rb
index 84ca827..2da1e22 100644
--- a/services/api/test/unit/job_test.rb
+++ b/services/api/test/unit/job_test.rb
@@ -60,6 +60,13 @@ class JobTest < ActiveSupport::TestCase
assert(job.invalid?, "Job with bad Docker tag valid")
end
+ test "locate a Docker image with a partial hash" do
+ image_hash = links(:docker_image_collection_hash).name[0..24]
+ job = Job.new(runtime_constraints: {'docker_image' => image_hash})
+ assert(job.valid?, "Job with partial Docker image hash failed")
+ assert_equal(collections(:docker_image).uuid, job.docker_image_locator)
+ end
+
{ 'name' => 'arvados_test_nonexistent',
'hash' => 'f' * 64,
'locator' => BAD_COLLECTION,
commit b64bddbe99166b02f979d7f47382308d31df7eee
Author: Brett Smith <brett at curoverse.com>
Date: Fri Jun 13 13:47:53 2014 -0400
2879: API server Jobs use new Docker image+tag link structure.
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index bf8119d..3073b07 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -117,16 +117,16 @@ class Job < ArvadosModel
links: {link_class: 'docker_image_hash'}).first
self.docker_image_locator = (coll.links_via_head.any?) ? coll.uuid : nil
else # Accept images specified by their Docker metadata.
- link_search = {name: image_name}
if image_name =~ /^[0-9A-Fa-f]{64}$/
- link_search[:link_class] = 'docker_image_hash'
+ link_search = {
+ link_class: 'docker_image_hash',
+ name: image_name,
+ }
else
- # Search for a match on image repository + tag.
- link_search[:link_class] = 'docker_image_repository'
- tag_name = runtime_constraints['docker_image_tag'] || 'latest'
- link_search[:head_uuid] = Link.select(:head_uuid).
- where(link_class: 'docker_image_tag', name: tag_name).
- map(&:head_uuid)
+ link_search = {
+ link_class: 'docker_image_repo+tag',
+ name: "#{image_name}:#{runtime_constraints['docker_image_tag'] || 'latest'}",
+ }
end
links = Link.where(link_search)
# Select the image that was created most recently.
diff --git a/services/api/test/fixtures/links.yml b/services/api/test/fixtures/links.yml
index cca2b26..d5496a6 100644
--- a/services/api/test/fixtures/links.yml
+++ b/services/api/test/fixtures/links.yml
@@ -479,8 +479,8 @@ docker_image_collection_tag:
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-06-11 14:30:00.184019565 Z
updated_at: 2014-06-11 14:30:00.183829316 Z
- link_class: docker_image_tag
- name: latest
+ link_class: docker_image_repo+tag
+ name: arvados/apitestfixture:latest
tail_uuid: ~
head_uuid: fa3c1a9cb6783f85f2ecda037e07b8c3+167
properties:
@@ -494,8 +494,8 @@ docker_image_collection_tag2:
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-06-11 14:30:00.184019565 Z
updated_at: 2014-06-11 14:30:00.183829316 Z
- link_class: docker_image_tag
- name: june10
+ link_class: docker_image_repo+tag
+ name: arvados/apitestfixture:june10
tail_uuid: ~
head_uuid: fa3c1a9cb6783f85f2ecda037e07b8c3+167
properties:
diff --git a/services/api/test/unit/job_test.rb b/services/api/test/unit/job_test.rb
index 3e9488c..84ca827 100644
--- a/services/api/test/unit/job_test.rb
+++ b/services/api/test/unit/job_test.rb
@@ -42,8 +42,8 @@ class JobTest < ActiveSupport::TestCase
end
test "locate a Docker image with a repository + tag" do
- image_repo = links(:docker_image_collection_repository).name
- image_tag = links(:docker_image_collection_tag2).name
+ image_repo, image_tag =
+ links(:docker_image_collection_tag2).name.split(':', 2)
job = Job.new(runtime_constraints:
{'docker_image' => image_repo,
'docker_image_tag' => image_tag})
commit 4c787bace891a9e1290fd64e75428929d8049989
Author: Brett Smith <brett at curoverse.com>
Date: Fri Jun 13 13:40:45 2014 -0400
2879: Store Docker image repo+tag together in one tag.
It doesn't make sense to query the tag independently of the
repository, and this change will make the common case of querying both
together easier.
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index abf60f2..236d9fc 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -204,7 +204,8 @@ def main(arguments=None):
make_link('docker_image_hash', image_hash, **link_base)
if not image_hash.startswith(args.image.lower()):
make_link('docker_image_repository', args.image, **link_base)
- make_link('docker_image_tag', args.tag, **link_base)
+ make_link('docker_image_repo+tag', '{}:{}'.format(args.image, args.tag),
+ **link_base)
# Clean up.
image_file.close()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list