[ARVADOS] updated: 051a3252f53a564ccac8d764cb0cff65b1987f77
Git user
git at public.curoverse.com
Fri Aug 19 17:03:37 EDT 2016
Summary of changes:
.../app/controllers/arvados/v1/jobs_controller.rb | 2 +-
services/api/app/models/job.rb | 16 +++++++++++++
...9195557_add_script_parameters_digest_to_jobs.rb | 6 +++++
...0819195725_populate_script_parameters_digest.rb | 21 ++++++++++++++++
services/api/db/structure.sql | 16 +++++++++++--
services/api/test/fixtures/jobs.yml | 28 ++++++++++++++++++++++
.../arvados/v1/job_reuse_controller_test.rb | 4 ++--
services/api/test/unit/job_test.rb | 16 +++++++++++++
8 files changed, 104 insertions(+), 5 deletions(-)
create mode 100644 services/api/db/migrate/20160819195557_add_script_parameters_digest_to_jobs.rb
create mode 100644 services/api/db/migrate/20160819195725_populate_script_parameters_digest.rb
via 051a3252f53a564ccac8d764cb0cff65b1987f77 (commit)
via 4a553d66354973620ec2f0970b028ca669448a4b (commit)
from 08344809b1887eb7c53e5bdff6171b4c731e5c72 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 051a3252f53a564ccac8d764cb0cff65b1987f77
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Aug 19 17:03:24 2016 -0400
9773: Use script_parameters_digest in reusable job query.
diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index d64fe01..de1d01c 100644
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -77,7 +77,7 @@ class Arvados::V1::JobsController < ApplicationController
readable_by(current_user).
where('state = ? or (owner_uuid = ? and state in (?))',
Job::Complete, current_user.uuid, [Job::Queued, Job::Running]).
- where('script_parameters = ?', resource_attrs[:script_parameters].to_yaml).
+ where('script_parameters_digest = ?', Job.sorted_hash_digest(resource_attrs[:script_parameters])).
where('nondeterministic is distinct from ?', true).
order('state desc, created_at') # prefer Running jobs over Queued
apply_filters
diff --git a/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb b/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
index 64d5591..ab79d7b 100644
--- a/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
@@ -19,8 +19,8 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
repository: "active/foo",
script_parameters: {
- input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
- an_integer: '1'
+ an_integer: '1',
+ input: 'fa7aeb5140e2848d39b416daeef4ffc5+45'
}
}
assert_response :success
commit 4a553d66354973620ec2f0970b028ca669448a4b
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Aug 19 17:02:03 2016 -0400
9773: Add script_parameters_digest column to support faster job reuse.
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index 0ed5353..4b9b687 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -11,6 +11,7 @@ class Job < ArvadosModel
after_commit :trigger_crunch_dispatch_if_cancelled, :on => :update
before_validation :set_priority
before_validation :update_state_from_old_state_attrs
+ before_validation :update_script_parameters_digest
validate :ensure_script_version_is_commit
validate :find_docker_image_locator
validate :find_arvados_sdk_version
@@ -105,8 +106,23 @@ class Job < ArvadosModel
end
end
+ def update_script_parameters_digest
+ self.script_parameters_digest = self.class.sorted_hash_digest(script_parameters)
+ end
+
protected
+ def self.sorted_hash_digest h
+ Digest::MD5.hexdigest(Oj.dump(deep_sort_hash(h)))
+ end
+
+ def self.deep_sort_hash h
+ return h unless h.is_a? Hash
+ h.sort.collect do |k, v|
+ [k, deep_sort_hash(v)]
+ end.to_h
+ end
+
def foreign_key_attributes
super + %w(output log)
end
diff --git a/services/api/db/migrate/20160819195557_add_script_parameters_digest_to_jobs.rb b/services/api/db/migrate/20160819195557_add_script_parameters_digest_to_jobs.rb
new file mode 100644
index 0000000..8ed3cfe
--- /dev/null
+++ b/services/api/db/migrate/20160819195557_add_script_parameters_digest_to_jobs.rb
@@ -0,0 +1,6 @@
+class AddScriptParametersDigestToJobs < ActiveRecord::Migration
+ def change
+ add_column :jobs, :script_parameters_digest, :string
+ add_index :jobs, :script_parameters_digest
+ end
+end
diff --git a/services/api/db/migrate/20160819195725_populate_script_parameters_digest.rb b/services/api/db/migrate/20160819195725_populate_script_parameters_digest.rb
new file mode 100644
index 0000000..fdfc97c
--- /dev/null
+++ b/services/api/db/migrate/20160819195725_populate_script_parameters_digest.rb
@@ -0,0 +1,21 @@
+class PopulateScriptParametersDigest < ActiveRecord::Migration
+ def up
+ done = false
+ while !done
+ done = true
+ Job.
+ where('script_parameters_digest is null').
+ select([:id, :script_parameters, :script_parameters_digest]).
+ limit(1000).
+ each do |j|
+ done = false
+ Job.
+ where('id=? or script_parameters=?', j.id, j.script_parameters.to_yaml).
+ update_all(script_parameters_digest: j.update_script_parameters_digest)
+ end
+ end
+ end
+
+ def down
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 1d7a2c6..bda4e5d 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -539,7 +539,8 @@ CREATE TABLE jobs (
description character varying(524288),
state character varying(255),
arvados_sdk_version character varying(255),
- components text
+ components text,
+ script_parameters_digest character varying(255)
);
@@ -1853,6 +1854,13 @@ CREATE INDEX index_jobs_on_script ON jobs USING btree (script);
--
+-- Name: index_jobs_on_script_parameters_digest; Type: INDEX; Schema: public; Owner: -; Tablespace:
+--
+
+CREATE INDEX index_jobs_on_script_parameters_digest ON jobs USING btree (script_parameters_digest);
+
+
+--
-- Name: index_jobs_on_started_at; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
@@ -2674,4 +2682,8 @@ INSERT INTO schema_migrations (version) VALUES ('20160509143250');
INSERT INTO schema_migrations (version) VALUES ('20160808151459');
-INSERT INTO schema_migrations (version) VALUES ('20160808151559');
\ No newline at end of file
+INSERT INTO schema_migrations (version) VALUES ('20160808151559');
+
+INSERT INTO schema_migrations (version) VALUES ('20160819195557');
+
+INSERT INTO schema_migrations (version) VALUES ('20160819195725');
\ No newline at end of file
diff --git a/services/api/test/fixtures/jobs.yml b/services/api/test/fixtures/jobs.yml
index 95cb967..584cc23 100644
--- a/services/api/test/fixtures/jobs.yml
+++ b/services/api/test/fixtures/jobs.yml
@@ -23,6 +23,7 @@ running:
done: 1
runtime_constraints: {}
state: Running
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
running_cancelled:
uuid: zzzzz-8i9sb-4cf0nhn6xte809j
@@ -49,6 +50,7 @@ running_cancelled:
done: 1
runtime_constraints: {}
state: Cancelled
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
uses_nonexistent_script_version:
uuid: zzzzz-8i9sb-7m339pu0x9mla88
@@ -75,6 +77,7 @@ uses_nonexistent_script_version:
done: 1
runtime_constraints: {}
state: Complete
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
foobar:
uuid: zzzzz-8i9sb-aceg2bnq7jt7kon
@@ -103,6 +106,7 @@ foobar:
done: 1
runtime_constraints: {}
state: Complete
+ script_parameters_digest: 03a43a7d84f7fb022467b876c2950acd
barbaz:
uuid: zzzzz-8i9sb-cjs4pklxxjykyuq
@@ -131,6 +135,7 @@ barbaz:
done: 1
runtime_constraints: {}
state: Complete
+ script_parameters_digest: c3d19d3ec50ac0914baa56b149640f73
runningbarbaz:
uuid: zzzzz-8i9sb-cjs4pklxxjykyuj
@@ -159,6 +164,7 @@ runningbarbaz:
done: 0
runtime_constraints: {}
state: Running
+ script_parameters_digest: c3d19d3ec50ac0914baa56b149640f73
previous_job_run:
uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
@@ -175,6 +181,7 @@ previous_job_run:
log: d41d8cd98f00b204e9800998ecf8427e+0
output: ea10d51bcf88862dbcc36eb292017dfd+45
state: Complete
+ script_parameters_digest: a5f03bbfb8ba88a2efe4a7852671605b
previous_ancient_job_run:
uuid: zzzzz-8i9sb-ahd7cie8jah9qui
@@ -191,6 +198,7 @@ previous_ancient_job_run:
log: d41d8cd98f00b204e9800998ecf8427e+0
output: ea10d51bcf88862dbcc36eb292017dfd+45
state: Complete
+ script_parameters_digest: 174dd339d44f2b259fadbab7ebdb8df9
previous_docker_job_run:
uuid: zzzzz-8i9sb-k6emstgk4kw4yhi
@@ -208,6 +216,7 @@ previous_docker_job_run:
output: ea10d51bcf88862dbcc36eb292017dfd+45
docker_image_locator: fa3c1a9cb6783f85f2ecda037e07b8c3+167
state: Complete
+ script_parameters_digest: a5f03bbfb8ba88a2efe4a7852671605b
previous_ancient_docker_image_job_run:
uuid: zzzzz-8i9sb-t3b460aolxxuldl
@@ -225,6 +234,7 @@ previous_ancient_docker_image_job_run:
output: ea10d51bcf88862dbcc36eb292017dfd+45
docker_image_locator: b519d9cb706a29fc7ea24dbea2f05851+93
state: Complete
+ script_parameters_digest: 174dd339d44f2b259fadbab7ebdb8df9
previous_job_run_with_arvados_sdk_version:
uuid: zzzzz-8i9sb-eoo0321or2dw2jg
@@ -244,6 +254,7 @@ previous_job_run_with_arvados_sdk_version:
success: true
output: ea10d51bcf88862dbcc36eb292017dfd+45
state: Complete
+ script_parameters_digest: a5f03bbfb8ba88a2efe4a7852671605b
previous_job_run_no_output:
uuid: zzzzz-8i9sb-cjs4pklxxjykppp
@@ -258,6 +269,7 @@ previous_job_run_no_output:
success: true
output: ~
state: Complete
+ script_parameters_digest: 174dd339d44f2b259fadbab7ebdb8df9
previous_job_run_superseded_by_hash_branch:
# This supplied_script_version is a branch name with later commits.
@@ -272,6 +284,7 @@ previous_job_run_superseded_by_hash_branch:
success: true
output: d41d8cd98f00b204e9800998ecf8427e+0
state: Complete
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
nondeterminisic_job_run:
uuid: zzzzz-8i9sb-cjs4pklxxjykyyy
@@ -286,6 +299,7 @@ nondeterminisic_job_run:
success: true
nondeterministic: true
state: Complete
+ script_parameters_digest: a5f03bbfb8ba88a2efe4a7852671605b
nearly_finished_job:
uuid: zzzzz-8i9sb-2gx6rz0pjl033w3
@@ -307,6 +321,7 @@ nearly_finished_job:
done: 0
runtime_constraints: {}
state: Complete
+ script_parameters_digest: 7ea26d58a79b7f5db9f90fb1e33d3006
queued:
uuid: zzzzz-8i9sb-grx15v5mjnsyxk7
@@ -329,6 +344,7 @@ queued:
tasks_summary: {}
runtime_constraints: {}
state: Queued
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
# A job with a log collection that can be parsed by the log viewer.
job_with_real_log:
@@ -338,6 +354,7 @@ job_with_real_log:
log: 0b9a7787660e1fce4a93f33e01376ba6+81
script_version: 7def43a4d3f20789dda4700f703b5514cc3ed250
state: Complete
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
cancelled:
uuid: zzzzz-8i9sb-4cf0abc123e809j
@@ -362,6 +379,7 @@ cancelled:
done: 1
runtime_constraints: {}
state: Cancelled
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
job_in_subproject:
uuid: zzzzz-8i9sb-subprojectjob01
@@ -372,6 +390,7 @@ job_in_subproject:
script: hash
script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
state: Complete
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
running_will_be_completed:
uuid: zzzzz-8i9sb-rshmckwoma9pjh8
@@ -396,6 +415,7 @@ running_will_be_completed:
done: 1
runtime_constraints: {}
state: Running
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
graph_stage1:
uuid: zzzzz-8i9sb-graphstage10000
@@ -405,6 +425,7 @@ graph_stage1:
script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
state: Complete
output: fa7aeb5140e2848d39b416daeef4ffc5+45
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
graph_stage2:
uuid: zzzzz-8i9sb-graphstage20000
@@ -417,6 +438,7 @@ graph_stage2:
input: fa7aeb5140e2848d39b416daeef4ffc5+45
input2: "stuff"
output: 65b17c95fdbc9800fc48acda4e9dcd0b+93
+ script_parameters_digest: 4900033ec5cfaf8a63566f3664aeaa70
graph_stage3:
uuid: zzzzz-8i9sb-graphstage30000
@@ -429,6 +451,7 @@ graph_stage3:
input: fa7aeb5140e2848d39b416daeef4ffc5+45
input2: "stuff2"
output: ea10d51bcf88862dbcc36eb292017dfd+45
+ script_parameters_digest: 02a085407e751d00b5dc88f1bd5e8247
job_with_latest_version:
uuid: zzzzz-8i9sb-nj8ioxnrvjtyk2b
@@ -458,6 +481,7 @@ job_with_latest_version:
done: 1
runtime_constraints: {}
state: Complete
+ script_parameters_digest: 03a43a7d84f7fb022467b876c2950acd
running_job_in_publicly_accessible_project:
uuid: zzzzz-8i9sb-n7omg50bvt0m1nf
@@ -470,6 +494,7 @@ running_job_in_publicly_accessible_project:
script_parameters:
input: fa7aeb5140e2848d39b416daeef4ffc5+45
input2: "stuff2"
+ script_parameters_digest: 02a085407e751d00b5dc88f1bd5e8247
completed_job_in_publicly_accessible_project:
uuid: zzzzz-8i9sb-jyq01m7in1jlofj
@@ -484,6 +509,7 @@ completed_job_in_publicly_accessible_project:
input2: "stuff2"
log: zzzzz-4zz18-4en62shvi99lxd4
output: b519d9cb706a29fc7ea24dbea2f05851+93
+ script_parameters_digest: 02a085407e751d00b5dc88f1bd5e8247
job_in_publicly_accessible_project_but_other_objects_elsewhere:
uuid: zzzzz-8i9sb-jyq01muyhgr4ofj
@@ -498,6 +524,7 @@ job_in_publicly_accessible_project_but_other_objects_elsewhere:
input2: "stuff2"
log: zzzzz-4zz18-fy296fx3hot09f7
output: zzzzz-4zz18-bv31uwvy3neko21
+ script_parameters_digest: 02a085407e751d00b5dc88f1bd5e8247
running_job_with_components:
uuid: zzzzz-8i9sb-with2components
@@ -527,3 +554,4 @@ running_job_with_components:
components:
component1: zzzzz-8i9sb-jyq01m7in1jlofj
component2: zzzzz-d1hrv-partdonepipelin
+ script_parameters_digest: 99914b932bd37a50b983c5e7c90ae93b
diff --git a/services/api/test/unit/job_test.rb b/services/api/test/unit/job_test.rb
index 832338a..1bc8035 100644
--- a/services/api/test/unit/job_test.rb
+++ b/services/api/test/unit/job_test.rb
@@ -440,4 +440,20 @@ class JobTest < ActiveSupport::TestCase
assert_equal('077ba2ad3ea24a929091a9e6ce545c93199b8e57',
internal_tag(j.uuid))
end
+
+ test 'script_parameters_digest is independent of key order' do
+ j1 = Job.new(job_attrs(script_parameters: {'a' => 'a', 'ddee' => {'d' => 'd', 'e' => 'e'}}))
+ j2 = Job.new(job_attrs(script_parameters: {'ddee' => {'e' => 'e', 'd' => 'd'}, 'a' => 'a'}))
+ assert j1.valid?
+ assert j2.valid?
+ assert_equal(j1.script_parameters_digest, j2.script_parameters_digest)
+ end
+
+ test 'job fixtures have correct script_parameters_digest' do
+ Job.all.each do |j|
+ d = j.script_parameters_digest
+ assert_equal(j.update_script_parameters_digest, d,
+ "wrong script_parameters_digest for #{j.uuid}")
+ end
+ end
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list