[ARVADOS] updated: 620bcbd5734e860a1e67bb85e4899233bc4a6b28
git at public.curoverse.com
git at public.curoverse.com
Wed Mar 19 16:33:06 EDT 2014
Summary of changes:
.../pipeline_instances/_show_components.html.erb | 10 ++
sdk/cli/bin/arv-run-pipeline-instance | 103 ++++---------------
.../app/controllers/arvados/v1/jobs_controller.rb | 11 ++-
services/api/config/environment.rb | 1 +
...0547_separate_repository_from_script_version.rb | 30 ++++++
services/api/db/schema.rb | 2 +-
6 files changed, 73 insertions(+), 84 deletions(-)
create mode 100644 services/api/db/migrate/20140319160547_separate_repository_from_script_version.rb
via 620bcbd5734e860a1e67bb85e4899233bc4a6b28 (commit)
via 3de8b7c50ba3d254acb888579f16de44ce693e36 (commit)
from 86c192b3ae3614f69305cfd60664c9b720b84692 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 620bcbd5734e860a1e67bb85e4899233bc4a6b28
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Mar 19 16:33:03 2014 -0400
Clear rails cache on startup.
diff --git a/services/api/config/environment.rb b/services/api/config/environment.rb
index 402d79f..9ef3a4a 100644
--- a/services/api/config/environment.rb
+++ b/services/api/config/environment.rb
@@ -4,3 +4,4 @@ require 'josh_id'
# Initialize the rails application
Server::Application.initialize!
+Rails.cache.clear
commit 3de8b7c50ba3d254acb888579f16de44ce693e36
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Mar 19 16:32:36 2014 -0400
* Added migration to break script_version into separate "repository" and "script_version" fields
* Added "no_reuse" which tells API server not to reuse old jobs but does not taint the job by setting the "nondeterministic" flag
* arv-run-pipeline-instance now creates jobs directly and relies on API server to determine when it is appropriate to re-use old jobs
diff --git a/apps/workbench/app/views/pipeline_instances/_show_components.html.erb b/apps/workbench/app/views/pipeline_instances/_show_components.html.erb
index 638fb8e..69dd62e 100644
--- a/apps/workbench/app/views/pipeline_instances/_show_components.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/_show_components.html.erb
@@ -68,6 +68,16 @@
<% content_for :js do %>
setInterval(function(){$('a.refresh').click()}, 15000);
<% end %>
+
+<% content_for :tab_line_buttons do %>
+ <%= form_tag @object, :method => :put do |f| %>
+
+ <%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => false %>
+
+ <%= button_tag "Stop pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
+ <% end %>
+<% end %>
+
<% end %>
<% else %>
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 56f0259..56e655f 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -28,11 +28,6 @@
# [--no-wait] Make only as much progress as possible without entering
# a sleep/poll loop.
#
-# [--no-reuse-finished] Do not reuse existing outputs to satisfy
-# pipeline components. Always submit a new job
-# or use an existing job which has not yet
-# finished.
-#
# [--no-reuse] Do not reuse existing jobs to satisfy pipeline
# components. Submit a new job for every component.
#
@@ -152,10 +147,6 @@ p = Trollop::Parser.new do
"Do not wait for jobs to finish. Just look up status, submit new jobs if needed, and exit.",
:short => :none,
:type => :boolean)
- opt(:no_reuse_finished,
- "Do not reuse existing outputs to satisfy pipeline components. Always submit a new job or use an existing job which has not yet finished.",
- :short => :none,
- :type => :boolean)
opt(:no_reuse,
"Do not reuse existing jobs to satisfy pipeline components. Submit a new job for every component.",
:short => :none,
@@ -331,7 +322,7 @@ class JobCache
if j.is_a? Hash and j[:uuid]
@cache[j[:uuid]] = j
else
- debuglog "create job: #{j[:errors] rescue nil}", 0
+ debuglog "create job: #{j[:errors] rescue nil} with attribute #{attributes}", 0
nil
end
end
@@ -433,85 +424,34 @@ class WhRunPipelineInstance
moretodo = false
@components.each do |cname, c|
job = nil
- if !c[:job] and
- c[:script_parameters].select { |pname, p| p.is_a? Hash }.empty?
- # Job is fully specified (all parameter values are present) but
- # no particular job has been found.
-
- debuglog "component #{cname} ready to satisfy."
-
- c.delete :wait
- second_place_job = nil # satisfies component, but not finished yet
-
- (@options[:no_reuse] ? [] : JobCache.
- where(script: c[:script],
- script_parameters: c[:script_parameters],
- script_version_descends_from: c[:script_version])
- ).each do |candidate_job|
- candidate_params_downcase = Hash[candidate_job[:script_parameters].
- map { |k,v| [k.downcase,v] }]
- c_params_downcase = Hash[c[:script_parameters].
- map { |k,v| [k.downcase,v] }]
-
- debuglog "component #{cname} considering job #{candidate_job[:uuid]} version #{candidate_job[:script_version]} parameters #{candidate_params_downcase.inspect}", 3
-
- unless candidate_params_downcase == c_params_downcase
- next
- end
-
- if c[:script_version] !=
- candidate_job[:script_version][0,c[:script_version].length]
- debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if script_version matched.", 2
- next
- end
- unless candidate_job[:success] || candidate_job[:running] ||
- (!candidate_job[:started_at] && !candidate_job[:cancelled_at])
- debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if it were running or successful.", 2
- next
- end
-
- if candidate_job[:success]
- unless @options[:no_reuse_finished] or c[:nondeterministic]
- job = candidate_job
- $stderr.puts "using #{job[:uuid]} (finished at #{job[:finished_at]}) for component #{cname}"
- c[:job] = job
- end
- else
- second_place_job ||= candidate_job
- end
- break
- end
- if not c[:job] and second_place_job
- job = second_place_job
- $stderr.puts "using #{job[:uuid]} (running since #{job[:started_at]}) for component #{cname}"
+ if !c[:job] and
+ c[:script_parameters].select { |pname, p| p.is_a? Hash and p[:output_of]}.empty?
+ # No job yet associated with this component and is component inputs
+ # are fully specified (any output_of script_parameters are resolved
+ # to real value)
+ job = JobCache.create({:script => c[:script],
+ :script_parameters => c[:script_parameters],
+ :script_version => c[:script_version],
+ :minimum_script_version => c[:minimum_script_version],
+ :exclude_script_versions => c[:exclude_minimum_script_versions],
+ :nondeterministic => c[:nondeterministic],
+ :no_reuse => @options[:no_reuse]})
+ if job
+ debuglog "component #{cname} new job #{job[:uuid]}"
c[:job] = job
+ else
+ debuglog "component #{cname} new job failed"
end
- if not c[:job]
- debuglog "component #{cname} not satisfied by any existing job."
- if !@options[:dry_run]
- debuglog "component #{cname} new job."
- job = JobCache.create(:script => c[:script],
- :script_parameters => c[:script_parameters],
- :runtime_constraints => c[:runtime_constraints] || {},
- :nondeterministic => c[:nondeterministic] || false,
- :script_version => c[:script_version] || 'master')
- if job
- debuglog "component #{cname} new job #{job[:uuid]}"
- c[:job] = job
- else
- debuglog "component #{cname} new job failed"
- end
- end
- end
- else
- c[:wait] = true
end
+
if c[:job] and c[:job][:uuid]
if (c[:job][:running] or
not (c[:job][:finished_at] or c[:job][:cancelled_at]))
- c[:job] = JobCache.get(c[:job][:uuid])
+ # Job is running so update copy of job record
+ c[:job] = JobCache.get(c[:job][:uuid])
end
+
if c[:job][:success]
# Populate script_parameters of other components waiting for
# this job
@@ -526,6 +466,7 @@ class WhRunPipelineInstance
end
elsif c[:job][:running] ||
(!c[:job][:started_at] && !c[:job][:cancelled_at])
+ # Job is still running
moretodo = true
elsif c[:job][:cancelled_at]
debuglog "component #{cname} job #{c[:job][:uuid]} cancelled."
diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index c8fcd22..a403356 100644
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -11,11 +11,16 @@ class Arvados::V1::JobsController < ApplicationController
resource_attrs[:minimum_script_version],
resource_attrs[:script_version],
resource_attrs[:exclude_script_versions])
- if !resource_attrs[:nondeterministic]
+ if !resource_attrs[:nondeterministic] and !resource_attrs[:no_reuse]
+ # Search for jobs where the script_version is in the list of commits
+ # returned by find_commit_range
Job.readable_by(current_user).where(script: resource_attrs[:script],
script_version: r).
each do |j|
- if j.nondeterministic != true and j.success != false and j.script_parameters == resource_attrs[:script_parameters]
+ if j.nondeterministic != true and
+ j.success != false and
+ j.script_parameters == resource_attrs[:script_parameters]
+ # We can re-use this job
@object = j
return show
end
@@ -26,8 +31,10 @@ class Arvados::V1::JobsController < ApplicationController
end
# Don't pass these on to activerecord
+ resource_attrs.delete(:repository)
resource_attrs.delete(:minimum_script_version)
resource_attrs.delete(:exclude_script_versions)
+ resource_attrs.delete(:no_reuse)
super
end
diff --git a/services/api/db/migrate/20140319160547_separate_repository_from_script_version.rb b/services/api/db/migrate/20140319160547_separate_repository_from_script_version.rb
new file mode 100644
index 0000000..6dac829
--- /dev/null
+++ b/services/api/db/migrate/20140319160547_separate_repository_from_script_version.rb
@@ -0,0 +1,30 @@
+class SeparateRepositoryFromScriptVersion < ActiveRecord::Migration
+ include CurrentApiClient
+
+ def fixup pt
+ c = pt.components
+ c.each do |k, v|
+ commit_ish = v["script_version"]
+ if commit_ish.index(':')
+ want_repo, commit_ish = commit_ish.split(':',2)
+ v[:repository] = want_repo
+ v[:script_version] = commit_ish
+ end
+ end
+ pt.save!
+ end
+
+ def up
+ act_as_system_user do
+ PipelineTemplate.all.each do |pt|
+ fixup pt
+ end
+ PipelineInstance.all.each do |pt|
+ fixup pt
+ end
+ end
+ end
+
+ def down
+ end
+end
diff --git a/services/api/db/schema.rb b/services/api/db/schema.rb
index 91d9106..4d638b5 100644
--- a/services/api/db/schema.rb
+++ b/services/api/db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20140317135600) do
+ActiveRecord::Schema.define(:version => 20140319160547) do
create_table "api_client_authorizations", :force => true do |t|
t.string "api_token", :null => false
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list