[ARVADOS] created: 9c8f3677ae05a7d8d25415d56757e362ffd45084
git at public.curoverse.com
git at public.curoverse.com
Sun Apr 20 20:09:06 EDT 2014
at 9c8f3677ae05a7d8d25415d56757e362ffd45084 (commit)
commit 9c8f3677ae05a7d8d25415d56757e362ffd45084
Author: radhika chippada <radhika at curoverse.com>
Date: Sun Apr 20 20:08:13 2014 -0400
Migration, pileline model updates, unit tests working.
diff --git a/apps/workbench/app/models/pipeline_instance.rb b/apps/workbench/app/models/pipeline_instance.rb
index ccb8835..ddbd09a 100644
--- a/apps/workbench/app/models/pipeline_instance.rb
+++ b/apps/workbench/app/models/pipeline_instance.rb
@@ -18,7 +18,7 @@ class PipelineInstance < ArvadosBase
end
def attribute_editable?(attr)
- attr.to_sym == :name || (attr.to_sym == :components and self.active == nil)
+ attr.to_sym == :name || (attr.to_sym == :components and self.state == 'New')
end
def attributes_for_display
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 7ddea30..13d2512 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -416,7 +416,7 @@ class WhRunPipelineInstance
@instance ||= PipelineInstance.
create(:components => @components,
:pipeline_template_uuid => @template[:uuid],
- :active => true)
+ :state => 'New')
self
end
@@ -449,6 +449,7 @@ class WhRunPipelineInstance
if job
debuglog "component #{cname} new job #{job[:uuid]}"
c[:job] = job
+ @instance[:state] = 'RunningOnClient' # correct?
else
debuglog "component #{cname} new job failed"
end
@@ -520,7 +521,9 @@ class WhRunPipelineInstance
end
end
@instance[:components] = @components
- @instance[:active] = moretodo
+ if moretodo
+ @instance[:state] = 'RunningOnClient' # correct?
+ end
report_status
if @options[:no_wait]
@@ -553,9 +556,18 @@ class WhRunPipelineInstance
end
end
+ # populate components_summary
+ @instance[:components_summary][:failed] = failed
+ @instance[:components_summary][:done] = ended
+ @instance[:components_summary][:todo] = @components.length - ended - failed
+ #@instance[:components_summary][:running] = TBD -- how to determine this?
+
if ended == @components.length or failed > 0
- @instance[:active] = false
- @instance[:success] = (succeeded == @components.length)
+ if succeeded == @components.length
+ @instance[:state] = 'Complete'
+ else
+ @instance[:state] = 'Failed'
+ end
end
@instance.save
@@ -563,7 +575,6 @@ class WhRunPipelineInstance
def cleanup
if @instance
- @instance[:active] = false
@instance.save
end
end
@@ -621,9 +632,11 @@ begin
runner.apply_parameters(p.leftovers)
runner.setup_instance
if $options[:submit]
+ runner.instance.set_state 'RunningOnServer'
runner.instance.save
puts runner.instance[:uuid]
else
+ runner.instance.set_state 'RunningOnClient'
runner.run
end
rescue Exception => e
diff --git a/services/api/app/models/pipeline_instance.rb b/services/api/app/models/pipeline_instance.rb
index b16769c..f18fa3a 100644
--- a/services/api/app/models/pipeline_instance.rb
+++ b/services/api/app/models/pipeline_instance.rb
@@ -4,10 +4,13 @@ class PipelineInstance < ArvadosModel
include CommonApiTemplate
serialize :components, Hash
serialize :properties, Hash
+ serialize :components_summary, Hash
belongs_to :pipeline_template, :foreign_key => :pipeline_template_uuid, :primary_key => :uuid
before_validation :bootstrap_components
before_validation :update_success
+ before_create :set_state_for_new_pipeline
+ before_save :set_state_for_new_pipeline
api_accessible :user, extend: :common do |t|
t.add :pipeline_template_uuid
@@ -16,14 +19,89 @@ class PipelineInstance < ArvadosModel
t.add :components
t.add :success
t.add :active
+ t.add :state
t.add :dependencies
t.add :properties
+ t.add :components_summary
end
+ # Supported states for a pipeline instance
+ New = 'New'
+ Ready = 'Ready'
+ RunningOnServer = 'RunningOnServer'
+ RunningOnClient = 'RunningOnClient'
+ Paused = 'Paused'
+ Failed = 'Failed'
+ Complete = 'Complete'
+
def dependencies
dependency_search(self.components).keys
end
+ def active
+ self.state == RunningOnServer || self.state == RunningOnClient
+ end
+
+ def success
+ self.state == Complete
+ end
+
+ def set_state state
+ self.state = state
+ end
+
+ def set_state_for_new_pipeline
+ if !self.state || self.state == New
+ if PipelineInstance.is_ready self.components
+ self.state = Ready
+ else
+ self.state = New
+ end
+ end
+ end
+
+ # if a legacy client tries to update active or success attributes, convert to state
+ def update_attribute name, value
+ if name == 'success'
+ if value == true
+ self.state = Complete
+ else
+ self.state = Failed
+ end
+
+ name = 'state'
+ value = self.state
+ elsif name == 'active'
+ if value == true
+ self.state = RunningOnServer
+ else
+ self.state = New
+ end
+
+ name = 'state'
+ value = self.state
+ end
+
+ super
+ end
+
+ # if all components have input, the pipeline is Ready
+ def self.is_ready components
+ if !components || components.empty? # is this correct?
+ return true
+ end
+
+ all_components_have_input = true
+ components.each do |name, component|
+ if !component.andand['script_parameters'].andand['input'] ||
+ component.andand['script_parameters'].andand['input'].empty?
+ all_components_have_input = false
+ break
+ end
+ end
+ return all_components_have_input
+ end
+
def progress_table
begin
# v0 pipeline format
@@ -40,7 +118,7 @@ class PipelineInstance < ArvadosModel
else
row << 0.0
if step['failed']
- self.success = false
+ self.state = Failed
end
end
row << (step['warehousejob']['id'] rescue nil)
@@ -61,7 +139,7 @@ class PipelineInstance < ArvadosModel
end
def self.queue
- self.where('active = true')
+ self.where("state = 'Ready' and state != 'RunningOnClient'")
end
protected
diff --git a/services/api/db/migrate/20140417195429_pipeline_instance_state.rb b/services/api/db/migrate/20140417195429_pipeline_instance_state.rb
new file mode 100644
index 0000000..6f43005
--- /dev/null
+++ b/services/api/db/migrate/20140417195429_pipeline_instance_state.rb
@@ -0,0 +1,45 @@
+class PipelineInstanceState < ActiveRecord::Migration
+ include CurrentApiClient
+
+ def up
+ add_column :pipeline_instances, :state, :string
+ add_column :pipeline_instances, :components_summary, :text
+
+ act_as_system_user do
+ PipelineInstance.all.each do |pi|
+ pi.state = PipelineInstance::New
+puts "\ninstance = #{pi.inspect}"
+puts "\nid = #{pi.id}"
+puts "\nsuccess = #{pi[:success]}"
+puts "\nactive = #{pi[:active]}"
+
+ if !pi.attribute_present? :success # success is nil
+ if pi[:active] == true
+ pi.state = PipelineInstance::RunningOnServer
+ else
+ if PipelineInstance.is_ready pi.components
+ pi.state = PipelineInstance::Ready
+ else
+ pi.state = PipelineInstance::New
+ end
+ end
+ elsif pi[:success] == true
+ pi.state = PipelineInstance::Complete
+ else
+ pi.state = PipelineInstance::Failed
+ end
+
+ pi.save!
+ end
+ end
+
+ remove_column :pipeline_instances, :active
+ remove_column :pipeline_instances, :success
+ end
+
+ def down
+ add_column :pipeline_instances, :success, :null => true
+ add_column :pipeline_instances, :active, :default => false
+ remove_column :pipeline_instances, :state
+ end
+end
diff --git a/services/api/db/schema.rb b/services/api/db/schema.rb
index e2301e5..cd8f00f 100644
--- a/services/api/db/schema.rb
+++ b/services/api/db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20140407184311) do
+ActiveRecord::Schema.define(:version => 20140417195429) do
create_table "api_client_authorizations", :force => true do |t|
t.string "api_token", :null => false
@@ -311,10 +311,10 @@ ActiveRecord::Schema.define(:version => 20140407184311) do
t.string "pipeline_template_uuid"
t.string "name"
t.text "components"
- t.boolean "success"
- t.boolean "active", :default => false
t.datetime "updated_at"
t.text "properties"
+ t.string "state"
+ t.text "components_summary"
end
add_index "pipeline_instances", ["created_at"], :name => "index_pipeline_instances_on_created_at"
diff --git a/services/api/test/fixtures/pipeline_instances.yml b/services/api/test/fixtures/pipeline_instances.yml
new file mode 100644
index 0000000..931eb68
--- /dev/null
+++ b/services/api/test/fixtures/pipeline_instances.yml
@@ -0,0 +1,3 @@
+new_pipeline:
+ uuid: zzzzz-xxxxx-f4gneyn6br1xize
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
diff --git a/services/api/test/unit/pipeline_instance_test.rb b/services/api/test/unit/pipeline_instance_test.rb
index 9b4c7c3..c6aae4d 100644
--- a/services/api/test/unit/pipeline_instance_test.rb
+++ b/services/api/test/unit/pipeline_instance_test.rb
@@ -1,7 +1,81 @@
require 'test_helper'
class PipelineInstanceTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+
+ test "check active and success for a pipeline in new state" do
+ pi = pipeline_instances :new_pipeline
+
+ assert !pi.active, 'expected active to be false for a new pipeline'
+ assert !pi.success, 'expected success to be false for a new pipeline'
+ assert !pi.state, 'expected state to be nil because the fixture had no state specified'
+
+ # save the pipeline and expect state to be New
+ pi.save
+ pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+ assert_equal PipelineInstance::Ready, pi.state, 'expected state to be New for new pipeline'
+ assert !pi.active, 'expected active to be false for a new pipeline'
+ assert !pi.success, 'expected success to be false for a new pipeline'
+ end
+
+ test "update attributes for pipeline" do
+ pi = pipeline_instances :new_pipeline
+
+ # add a component with no input and expect state to be New
+ component = {'script_parameters' => {"something" => "c1bad4b39ca5a924e481008009d94e32+210"}}
+ pi.components['first'] = component
+ components = pi.components
+ pi.update_attribute 'components', pi.components
+ pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+ assert_equal PipelineInstance::New, pi.state, 'expected state to be Ready after adding component with input'
+ assert_equal pi.components.size, 1, 'expected one component'
+ assert !pi.active, 'expected active to be false after update'
+ assert !pi.success, 'expected success to be false for a new pipeline'
+
+ # add a component with input and expect state to become Ready
+ component = {'script_parameters' => {"input" => "yyyad4b39ca5a924e481008009d94e32+210"}}
+ pi.components['first'] = component
+ components = pi.components
+ pi.update_attribute 'components', pi.components
+ pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+ assert_equal PipelineInstance::Ready, pi.state, 'expected state to be Ready after adding component with input'
+ assert_equal pi.components.size, 1, 'expected one component'
+ assert !pi.active, 'expected active to be false after update'
+ assert !pi.success, 'expected success to be false for a new pipeline'
+
+ pi.update_attribute 'active', true
+ pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+ assert_equal PipelineInstance::RunningOnServer, pi.state, 'expected state to be RunningOnServer after updating active to true'
+ assert pi.active, 'expected active to be true after update'
+ assert !pi.success, 'expected success to be false for a new pipeline'
+
+ pi.update_attribute 'success', false
+ pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+ assert_equal PipelineInstance::Failed, pi.state, 'expected state to be Failed after updating success to false'
+ assert !pi.active, 'expected active to be false after update'
+ assert !pi.success, 'expected success to be false for a new pipeline'
+
+ pi.update_attribute 'success', true
+ pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+ assert_equal PipelineInstance::Complete, pi.state, 'expected state to be Complete after updating success to true'
+ assert !pi.active, 'expected active to be false after update'
+ assert pi.success, 'expected success to be true after update'
+ end
+
+ test "update attributes for pipeline with two components" do
+ pi = pipeline_instances :new_pipeline
+
+ # add two components, one with input and one with no input and expect state to be New
+ component1 = {'script_parameters' => {"something" => "xxxad4b39ca5a924e481008009d94e32+210", "input" => "c1bad4b39ca5a924e481008009d94e32+210"}}
+ component2 = {'script_parameters' => {"something_else" => "xxxad4b39ca5a924e481008009d94e32+210", "not_input" => "c1bad4b39ca5a924e481008009d94e32+210"}}
+ pi.components['first'] = component1
+ pi.components['second'] = component2
+ components = pi.components
+ pi.update_attribute 'components', pi.components
+ pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+ assert_equal PipelineInstance::New, pi.state, 'expected state to be Ready after adding component with input'
+ assert_equal pi.components.size, 2, 'expected two components'
+ assert !pi.active, 'expected active to be false after update'
+ assert !pi.success, 'expected success to be false for a new pipeline'
+ end
+
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list