[ARVADOS] created: 1f2a831876fe1fd193c27639078d2de43cfd691b
git at public.curoverse.com
git at public.curoverse.com
Mon Jun 16 18:00:42 EDT 2014
at 1f2a831876fe1fd193c27639078d2de43cfd691b (commit)
commit 1f2a831876fe1fd193c27639078d2de43cfd691b
Author: Brett Smith <brett at curoverse.com>
Date: Mon Jun 16 18:01:40 2014 -0400
2879: API server can find_or_create Jobs based on Docker image.
diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index 6fddba7..a93167a 100644
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -28,27 +28,26 @@ class Arvados::V1::JobsController < ApplicationController
end
if params[:find_or_create]
- # Convert old special-purpose creation parameters to the new
- # filters-based method.
+ # Translate older creation parameters and special range operators
+ # into standard filters.
minimum_script_version = params[:minimum_script_version]
exclude_script_versions = params.fetch(:exclude_script_versions, [])
- @filters.select do |(col_name, operand, operator)|
- case col_name
- when "script_version"
- case operand
- when "in range"
- minimum_script_version = operator
- false
- when "not in", "not in range"
- begin
- exclude_script_versions += operator
- rescue TypeError
- exclude_script_versions << operator
- end
- false
- else
- true
+ @filters.select do |filter|
+ case filter[0..1]
+ when ["script_version", "in range"]
+ minimum_script_version = filter.last
+ false
+ when ["script_version", "not in"], ["script_version", "not in range"]
+ begin
+ exclude_script_versions += filter.last
+ rescue TypeError
+ exclude_script_versions << filter.last
end
+ false
+ when ["docker_image_locator", "in range"], ["docker_image_locator", "not in range"]
+ filter[1].sub!(/ range$/, '')
+ filter[2] = Collection.uuids_for_docker_image(filter[2])
+ true
else
true
end
@@ -64,7 +63,19 @@ class Arvados::V1::JobsController < ApplicationController
if @filters.select { |f| f.first == "script" }.empty?
@filters.append(["script", "=", resource_attrs[:script]])
end
+ if @filters.select { |f| f.first == "docker_image_locator" }.empty?
+ if image_search = resource_attrs[:runtime_constraints].andand["docker_image"]
+ image_tag = resource_attrs[:runtime_constraints]["docker_image_tag"]
+ image_locator =
+ Collection.uuids_for_docker_image(image_search, image_tag).last
+ return super if image_locator.nil? # We won't find anything to reuse.
+ @filters.append(["docker_image_locator", "=", image_locator])
+ else
+ @filters.append(["docker_image_locator", "=", nil])
+ end
+ end
+ # Search for a reusable Job, and return it if found.
@objects = Job.readable_by(current_user)
apply_filters
@object = nil
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 64a6bb0..2d573e5 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -148,7 +148,7 @@ class Collection < ArvadosModel
[hash_part, size_part].compact.join '+'
end
- def self.for_latest_docker_image(search_term, search_tag=nil, readers=nil)
+ def self.uuids_for_docker_image(search_term, search_tag=nil, readers=nil)
readers ||= [Thread.current[:user]]
base_search = Link.
readable_by(*readers).
@@ -161,7 +161,7 @@ class Collection < ArvadosModel
coll_matches = base_search.
where(link_class: "docker_image_hash", collections: {uuid: search_term})
if match = coll_matches.first
- return find_by_uuid(match.head_uuid)
+ return [match.head_uuid]
end
# Find Collections with matching Docker image repository+tag pairs.
@@ -176,20 +176,27 @@ class Collection < ArvadosModel
"docker_image_hash", "#{search_term}%")
end
- # Select the image that was created most recently. Note that the
- # SQL search order and fallback timestamp values are chosen so
- # that if image timestamps are missing, we use the image with the
- # newest link.
- latest_image_link = nil
- latest_image_timestamp = "1900-01-01T00:00:00Z"
+ # Generate an order key for each result. We want to order the results
+ # so that anything with an image timestamp is considered more recent than
+ # anything without; then we use the link's created_at as a tiebreaker.
+ results = {}
matches.find_each do |link|
- link_timestamp = link.properties.fetch("image_timestamp",
- "1900-01-01T00:00:01Z")
- if link_timestamp > latest_image_timestamp
- latest_image_link = link
- latest_image_timestamp = link_timestamp
+ sort_key = []
+ if timestamp = link.properties["image_timestamp"]
+ sort_key.push("Z", timestamp.to_s)
end
+ sort_key.push("Y", link.created_at.to_s(:db))
+ results[link] = sort_key.join("")
+ end
+ results.keys.sort_by { |link| results[link] }.map { |link| link.head_uuid }
+ end
+
+ def self.for_latest_docker_image(search_term, search_tag=nil, readers=nil)
+ image_uuid = uuids_for_docker_image(search_term, search_tag, readers).last
+ if image_uuid.nil?
+ nil
+ else
+ find_by_uuid(image_uuid)
end
- latest_image_link.nil? ? nil : find_by_uuid(latest_image_link.head_uuid)
end
end
diff --git a/services/api/test/fixtures/jobs.yml b/services/api/test/fixtures/jobs.yml
index 3ad7746..c9b2588 100644
--- a/services/api/test/fixtures/jobs.yml
+++ b/services/api/test/fixtures/jobs.yml
@@ -125,6 +125,18 @@ previous_job_run:
success: true
output: ea10d51bcf88862dbcc36eb292017dfd+45
+previous_docker_job_run:
+ uuid: zzzzz-8i9sb-k6emstgk4kw4yhi
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ script: hash
+ script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
+ script_parameters:
+ input: fa7aeb5140e2848d39b416daeef4ffc5+45
+ an_integer: "1"
+ success: true
+ output: ea10d51bcf88862dbcc36eb292017dfd+45
+ docker_image_locator: fa3c1a9cb6783f85f2ecda037e07b8c3+167
+
previous_job_run_no_output:
uuid: zzzzz-8i9sb-cjs4pklxxjykppp
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
diff --git a/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb b/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
index 363c468..e534adc 100644
--- a/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
@@ -342,4 +342,73 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
end
+
+ test "can reuse a Job with a Docker image" do
+ post(:create, {
+ job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ },
+ runtime_constraints: {
+ docker_image: 'arvados/apitestfixture',
+ }
+ },
+ find_or_create: true,
+ })
+ assert_response :success
+ new_job = assigns(:object)
+ assert_not_nil new_job
+ target_job = jobs(:previous_docker_job_run)
+ [:uuid, :script_version, :docker_image_locator].each do |attr|
+ assert_equal(target_job.send(attr), new_job.send(attr))
+ end
+ end
+
+ test "can reuse a Job with a Docker image hash filter" do
+ post(:create, {
+ job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ },
+ },
+ filters: [["docker_image_locator", "in range",
+ links(:docker_image_collection_hash).name]],
+ find_or_create: true,
+ })
+ assert_response :success
+ new_job = assigns(:object)
+ assert_not_nil new_job
+ target_job = jobs(:previous_docker_job_run)
+ [:uuid, :script_version, :docker_image_locator].each do |attr|
+ assert_equal(target_job.send(attr), new_job.send(attr))
+ end
+ end
+
+ test "new job with unknown Docker image filter" do
+ post(:create, {
+ job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ },
+ },
+ filters: [["docker_image_locator", "in range", "_nonexistentname_"]],
+ find_or_create: true,
+ })
+ assert_response :success
+ new_job = assigns(:object)
+ assert_not_nil new_job
+ assert_not_equal(jobs(:previous_docker_job_run).uuid, new_job.uuid)
+ end
end
commit a157275dd8b529ce089cd9f126fb6a0e89e7f456
Author: Brett Smith <brett at curoverse.com>
Date: Mon Jun 16 16:49:15 2014 -0400
2879: API server can find_or_create Jobs based on filters.
This will make it easier to support new criteria for find_or_create.
In particular, this will be the basis for searching based on Docker
image.
diff --git a/services/api/app/controllers/application_controller.rb b/services/api/app/controllers/application_controller.rb
index 9a54abe..fe5598e 100644
--- a/services/api/app/controllers/application_controller.rb
+++ b/services/api/app/controllers/application_controller.rb
@@ -130,14 +130,17 @@ class ApplicationController < ActionController::Base
apply_where_limit_order_params
end
- def apply_where_limit_order_params
- ar_table_name = @objects.table_name
-
- ft = record_filters @filters, ar_table_name
+ def apply_filters
+ ft = record_filters @filters, @objects.table_name
if ft[:cond_out].any?
@objects = @objects.where(ft[:cond_out].join(' AND '), *ft[:param_out])
end
+ end
+ def apply_where_limit_order_params
+ apply_filters
+
+ ar_table_name = @objects.table_name
if @where.is_a? Hash and @where.any?
conditions = ['1=1']
@where.each do |attr,value|
diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index ffa78b9..6fddba7 100644
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -13,6 +13,7 @@ class Arvados::V1::JobsController < ApplicationController
}, status: :unprocessable_entity
end
end
+ load_filters_param
# We used to ask for the minimum_, exclude_, and no_reuse params
# in the job resource. Now we advertise them as flags that alter
@@ -27,18 +28,48 @@ class Arvados::V1::JobsController < ApplicationController
end
if params[:find_or_create]
- r = Commit.find_commit_range(current_user,
- resource_attrs[:repository],
- params[:minimum_script_version],
- resource_attrs[:script_version],
- params[:exclude_script_versions])
- # Search for jobs whose script_version is in the list of commits
- # returned by find_commit_range
+ # Convert old special-purpose creation parameters to the new
+ # filters-based method.
+ minimum_script_version = params[:minimum_script_version]
+ exclude_script_versions = params.fetch(:exclude_script_versions, [])
+ @filters.select do |(col_name, operand, operator)|
+ case col_name
+ when "script_version"
+ case operand
+ when "in range"
+ minimum_script_version = operator
+ false
+ when "not in", "not in range"
+ begin
+ exclude_script_versions += operator
+ rescue TypeError
+ exclude_script_versions << operator
+ end
+ false
+ else
+ true
+ end
+ else
+ true
+ end
+ end
+ @filters.append(["script_version", "in",
+ Commit.find_commit_range(current_user,
+ resource_attrs[:repository],
+ minimum_script_version,
+ resource_attrs[:script_version],
+ exclude_script_versions)])
+
+ # Set up default filters for specific parameters.
+ if @filters.select { |f| f.first == "script" }.empty?
+ @filters.append(["script", "=", resource_attrs[:script]])
+ end
+
+ @objects = Job.readable_by(current_user)
+ apply_filters
@object = nil
incomplete_job = nil
- Job.readable_by(current_user).where(script: resource_attrs[:script],
- script_version: r).
- each do |j|
+ @objects.each do |j|
if j.nondeterministic != true and
((j.success == true and j.output != nil) or j.running == true) and
j.script_parameters == resource_attrs[:script_parameters]
diff --git a/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb b/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
index bfecf54..363c468 100644
--- a/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
@@ -278,4 +278,68 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
end
+ test "can reuse a Job based on filters" do
+ post(:create, {
+ job: {
+ script: "hash",
+ script_version: "master",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ },
+ filters: [["script_version", "in range", "tag1"]],
+ find_or_create: true,
+ })
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
+
+ test "can not reuse a Job based on filters" do
+ post(:create, {
+ job: {
+ script: "hash",
+ script_version: "master",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ },
+ filters: [["script_version", "in range",
+ "31ce37fe365b3dc204300a3e4c396ad333ed0556"],
+ ["script_version", "not in", ["tag1"]]],
+ find_or_create: true,
+ })
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
+ end
+
+ test "can not reuse a Job based on arbitrary filters" do
+ post(:create, {
+ job: {
+ script: "hash",
+ script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
+ script_parameters: {
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
+ an_integer: '1'
+ }
+ },
+ filters: [["created_at", "<", "2010-01-01T00:00:00Z"]],
+ find_or_create: true,
+ })
+ assert_response :success
+ assert_not_nil assigns(:object)
+ new_job = JSON.parse(@response.body)
+ assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
+ assert_equal '4fe459abe02d9b365932b8f5dc419439ab4e2577', new_job['script_version']
+ end
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list