[ARVADOS] created: 2f69408f572f6615504a6c17aba8c27ecba031d6
git at public.curoverse.com
git at public.curoverse.com
Fri Nov 14 13:57:38 EST 2014
at 2f69408f572f6615504a6c17aba8c27ecba031d6 (commit)
commit 2f69408f572f6615504a6c17aba8c27ecba031d6
Author: Brett Smith <brett at curoverse.com>
Date: Fri Nov 14 13:57:33 2014 -0500
4027: API server considers arvados_sdk_version for Job reuse.
* Extend "in git" and "not in git" filters to the arvados_sdk_version
column.
* When no filters are specified, if the submitted job specifies an
arvados_sdk_version constraint, require reuse candidates to meet it.
* Document all this.
diff --git a/doc/api/methods/jobs.html.textile.liquid b/doc/api/methods/jobs.html.textile.liquid
index 539a4a0..ac68129 100644
--- a/doc/api/methods/jobs.html.textile.liquid
+++ b/doc/api/methods/jobs.html.textile.liquid
@@ -45,9 +45,9 @@ h3. Specialized filters
Special filter operations are available for specific Job columns.
-* @script_version@ @in git@ @REFSPEC@<br>Resolve @REFSPEC@ to a list of git commits, and match jobs with a @script_version@ in that list. The create method will find commits between @REFSPEC@ and the submitted job's @script_version@; the list method will search between @REFSPEC@ and HEAD. This list may include parallel branches if there is more than one path between @REFSPEC@ and the end commit in the graph. Use @not in@ or @not in git@ filters (below) to blacklist specific commits.
+* @script_version@ @in git@ @REFSPEC@, @arvados_sdk_version@ @in git@ @REFSPEC@<br>Resolve @REFSPEC@ to a list of Git commits, and match jobs with a @script_version@ or @arvados_sdk_version@ in that list. When creating a job and filtering @script_version@, the search will find commits between @REFSPEC@ and the submitted job's @script_version@; all other searches will find commits between @REFSPEC@ and HEAD. This list may include parallel branches if there is more than one path between @REFSPEC@ and the end commit in the graph. Use @not in@ or @not in git@ filters (below) to blacklist specific commits.
-* @script_version@ @not in git@ @REFSPEC@<br>Resolve @REFSPEC@ to a list of git commits, and match jobs with a @script_version@ not in that list.
+* @script_version@ @not in git@ @REFSPEC@, @arvados_sdk_version@ @not in git@ @REFSPEC@<br>Resolve @REFSPEC@ to a list of Git commits, and match jobs with a @script_version@ or @arvados_sdk_version@ not in that list.
* @docker_image_locator@ @in docker@ @SEARCH@<br>@SEARCH@ can be a Docker image hash, a repository name, or a repository name and tag separated by a colon (@:@). The server will find collections that contain a Docker image that match that search criteria, then match jobs with a @docker_image_locator@ in that list.
@@ -61,7 +61,7 @@ notextile. <div class="spaced-out">
# If @find_or_create@ is false or omitted, create a new job and skip the rest of these steps.
# If @filters@ are specified, find jobs that match those filters. Filters *must* be specified to limit the @repository@ and @script@ attributes. An error is returned if they are missing.
-# If @filters@ are not specified, find jobs with the same @repository@ and @script@, with a @script_version@ between @minimum_script_version@ and @script_version@ (excluding @excluded_script_versions@), and a @docker_image_locator@ with the latest Collection that matches the submitted job's @docker_image@ constraint.
+# If @filters@ are not specified, find jobs with the same @repository@ and @script@, with a @script_version@ between @minimum_script_version@ and @script_version@ (excluding @excluded_script_versions@), and a @docker_image_locator@ with the latest Collection that matches the submitted job's @docker_image@ constraint. If the submitted job includes an @arvados_sdk_version@ constraint, jobs must have an @arvados_sdk_version@ between that refspec and HEAD to be found.
# If the found jobs include a completed job, and all found completed jobs have consistent output, return one of them. Which specific job is returned is undefined.
# If the found jobs only include incomplete jobs, return one of them. Which specific job is returned is undefined.
# If no job has been returned so far, create and return a new job.
diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index e2bef88..76528d0 100644
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -43,6 +43,9 @@ class Arvados::V1::JobsController < ApplicationController
else
@filters.append(["docker_image_locator", "=", nil])
end
+ if sdk_version = resource_attrs[:runtime_constraints].andand["arvados_sdk_version"]
+ @filters.append(["arvados_sdk_version", "in git", sdk_version])
+ end
begin
load_job_specific_filters
rescue ArgumentError => error
@@ -196,59 +199,73 @@ class Arvados::V1::JobsController < ApplicationController
def load_job_specific_filters
# Convert Job-specific @filters entries into general SQL filters.
script_info = {"repository" => nil, "script" => nil}
- script_range = {"exclude_versions" => []}
- @filters.select! do |filter|
- if (script_info.has_key? filter[0]) and (filter[1] == "=")
- if script_info[filter[0]].nil?
- script_info[filter[0]] = filter[2]
- elsif script_info[filter[0]] != filter[2]
- raise ArgumentError.new("incompatible #{filter[0]} filters")
+ git_filters = Hash.new do |hash, key|
+ hash[key] = {"max_version" => "HEAD", "exclude_versions" => []}
+ end
+ @filters.select! do |(col, op, val)|
+ if (script_info.has_key? col) and (op == "=")
+ if script_info[col].nil?
+ script_info[col] = val
+ elsif script_info[col] != val
+ raise ArgumentError.new("incompatible #{col} filters")
end
end
- case filter[0..1]
- when ["script_version", "in git"]
- script_range["min_version"] = filter.last
+ case op
+ when "in git"
+ git_filters[col]["min_version"] = val
false
- when ["script_version", "not in git"]
+ when "not in git"
begin
- script_range["exclude_versions"] += filter.last
+ git_filters[col]["exclude_versions"] += val
rescue TypeError
- script_range["exclude_versions"] << filter.last
+ git_filters[col]["exclude_versions"] << val
end
false
- when ["docker_image_locator", "in docker"], ["docker_image_locator", "not in docker"]
- filter[1].sub!(/ docker$/, '')
- search_list = filter[2].is_a?(Enumerable) ? filter[2] : [filter[2]]
- filter[2] = search_list.flat_map do |search_term|
+ when "in docker", "not in docker"
+ search_list = val.is_a?(Enumerable) ? val : [val]
+ image_hashes = search_list.flat_map do |search_term|
image_search, image_tag = search_term.split(':', 2)
Collection.find_all_for_docker_image(image_search, image_tag, @read_users).map(&:portable_data_hash)
end
- true
+ @filters << [col, op.sub(/ docker$/, ""), image_hashes]
+ false
else
true
end
end
# Build a real script_version filter from any "not? in git" filters.
- if (script_range.size > 1) or script_range["exclude_versions"].any?
- script_info.each_pair do |key, value|
- if value.nil?
- raise ArgumentError.new("script_version filter needs #{key} filter")
+ git_filters.each_pair do |col, filter|
+ case col
+ when "script_version"
+ script_info.each_pair do |key, value|
+ if value.nil?
+ raise ArgumentError.new("script_version filter needs #{key} filter")
+ end
+ end
+ filter["repository"] = script_info["repository"]
+ begin
+ filter["max_version"] = resource_attrs[:script_version]
+ rescue
+ # Using the HEAD default is fine.
end
+ when "arvados_sdk_version"
+ filter["repository"] = "arvados"
+ else
+ raise ArgumentError.new("unknown column for git filter: #{col}")
end
- last_version = begin resource_attrs[:script_version] rescue "HEAD" end
version_range = Commit.find_commit_range(current_user,
- script_info["repository"],
- script_range["min_version"],
- last_version,
- script_range["exclude_versions"])
+ filter["repository"],
+ filter["min_version"],
+ filter["max_version"],
+ filter["exclude_versions"])
if version_range.nil?
raise ArgumentError.
- new(["error searching #{script_info['repository']} from",
- "'#{script_range['min_version']}' to '#{last_version}',",
- "excluding #{script_range['exclude_versions']}"].join(" "))
+ new("error searching #{filter['repository']} from " +
+ "'#{filter['min_version']}' to '#{filter['max_version']}', " +
+ "excluding #{filter['exclude_versions']}")
end
- @filters.append(["script_version", "in", version_range])
+ @filters.append([col, "in", version_range])
end
end
diff --git a/services/api/test/fixtures/jobs.yml b/services/api/test/fixtures/jobs.yml
index 6721c12..888cb2a 100644
--- a/services/api/test/fixtures/jobs.yml
+++ b/services/api/test/fixtures/jobs.yml
@@ -183,6 +183,23 @@ previous_docker_job_run:
docker_image_locator: fa3c1a9cb6783f85f2ecda037e07b8c3+167
state: Complete
+previous_job_run_with_arvados_sdk_version:
+ uuid: zzzzz-8i9sb-eoo0321or2dw2jg
+ created_at: <%= 14.minute.ago.to_s(:db) %>
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ repository: foo
+ script: hash
+ script_version: 31ce37fe365b3dc204300a3e4c396ad333ed0556
+ script_parameters:
+ input: fa7aeb5140e2848d39b416daeef4ffc5+45
+ an_integer: "1"
+ runtime_constraints:
+ arvados_sdk_version: commit2
+ arvados_sdk_version: 00634b2b8a492d6f121e3cf1d6587b821136a9a7
+ success: true
+ output: ea10d51bcf88862dbcc36eb292017dfd+45
+ state: Complete
+
previous_job_run_no_output:
uuid: zzzzz-8i9sb-cjs4pklxxjykppp
created_at: <%= 14.minute.ago.to_s(:db) %>
diff --git a/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb b/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
index 01f2b9c..d08fbbf 100644
--- a/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
@@ -294,7 +294,8 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
assert_not_nil assigns(:object)
new_job = JSON.parse(@response.body)
assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
- assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
+ assert_not_equal('4fe459abe02d9b365932b8f5dc419439ab4e2577',
+ new_job['script_version'])
end
test "cannot reuse job with find_or_create but excluded version" do
@@ -316,7 +317,8 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
assert_not_nil assigns(:object)
new_job = JSON.parse(@response.body)
assert_not_equal 'zzzzz-8i9sb-cjs4pklxxjykqqq', new_job['uuid']
- assert_equal '077ba2ad3ea24a929091a9e6ce545c93199b8e57', new_job['script_version']
+ assert_not_equal('4fe459abe02d9b365932b8f5dc419439ab4e2577',
+ new_job['script_version'])
end
BASE_FILTERS = {
@@ -324,6 +326,7 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
'script' => ['=', 'hash'],
'script_version' => ['in git', 'master'],
'docker_image_locator' => ['=', nil],
+ 'arvados_sdk_version' => ['=', nil],
}
def filters_from_hash(hash)
@@ -663,4 +666,32 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
assert(errors.any? { |msg| msg.include? "__nosuchbranch__" },
"bad refspec not mentioned in error message")
end
+
+ test "can't reuse job with older Arvados SDK version" do
+ params = {
+ script_version: "31ce37fe365b3dc204300a3e4c396ad333ed0556",
+ runtime_constraints: {"arvados_sdk_version" => "master"},
+ }
+ check_new_job_created_from(job: params)
+ end
+
+ test "reuse job from arvados_sdk_version git filters" do
+ filter_h = BASE_FILTERS.
+ merge("arvados_sdk_version" => ["in git", "commit2"])
+ filter_h.delete("script_version")
+ params = create_foo_hash_job_params(filters: filters_from_hash(filter_h))
+ post(:create, params)
+ assert_response :success
+ assert_equal(jobs(:previous_job_run_with_arvados_sdk_version).uuid,
+ assigns(:object).uuid)
+ end
+
+ test "create new job because of arvados_sdk_version 'not in git' filters" do
+ filter_h = BASE_FILTERS.reject { |k| k == "script_version" }
+ filter_a = filters_from_hash(filter_h)
+ # Allow anything from the root commit, but before commit 2.
+ filter_a += [["arvados_sdk_version", "in git", "436637c8"],
+ ["arvados_sdk_version", "not in git", "00634b2b"]]
+ check_new_job_created_from(filters: filter_a)
+ end
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list