[ARVADOS] created: 9c8f3677ae05a7d8d25415d56757e362ffd45084

git at public.curoverse.com git at public.curoverse.com
Sun Apr 20 20:09:06 EDT 2014


        at  9c8f3677ae05a7d8d25415d56757e362ffd45084 (commit)


commit 9c8f3677ae05a7d8d25415d56757e362ffd45084
Author: radhika chippada <radhika at curoverse.com>
Date:   Sun Apr 20 20:08:13 2014 -0400

    Migration, pileline model updates, unit tests working.

diff --git a/apps/workbench/app/models/pipeline_instance.rb b/apps/workbench/app/models/pipeline_instance.rb
index ccb8835..ddbd09a 100644
--- a/apps/workbench/app/models/pipeline_instance.rb
+++ b/apps/workbench/app/models/pipeline_instance.rb
@@ -18,7 +18,7 @@ class PipelineInstance < ArvadosBase
   end
   
   def attribute_editable?(attr)
-    attr.to_sym == :name || (attr.to_sym == :components and self.active == nil)
+    attr.to_sym == :name || (attr.to_sym == :components and self.state == 'New')
   end
 
   def attributes_for_display
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 7ddea30..13d2512 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -416,7 +416,7 @@ class WhRunPipelineInstance
     @instance ||= PipelineInstance.
       create(:components => @components,
              :pipeline_template_uuid => @template[:uuid],
-             :active => true)
+             :state => 'New')
     self
   end
 
@@ -449,6 +449,7 @@ class WhRunPipelineInstance
           if job
             debuglog "component #{cname} new job #{job[:uuid]}"
             c[:job] = job
+            @instance[:state] = 'RunningOnClient'  # correct?
           else
             debuglog "component #{cname} new job failed"
           end
@@ -520,7 +521,9 @@ class WhRunPipelineInstance
         end
       end
       @instance[:components] = @components
-      @instance[:active] = moretodo
+      if moretodo
+        @instance[:state] = 'RunningOnClient'  # correct?
+      end
       report_status
 
       if @options[:no_wait]
@@ -553,9 +556,18 @@ class WhRunPipelineInstance
       end
     end
 
+    # populate components_summary
+    @instance[:components_summary][:failed] = failed
+    @instance[:components_summary][:done] = ended
+    @instance[:components_summary][:todo] = @components.length - ended - failed
+    #@instance[:components_summary][:running] = TBD -- how to determine this?
+    
     if ended == @components.length or failed > 0
-      @instance[:active] = false
-      @instance[:success] = (succeeded == @components.length)
+      if succeeded == @components.length
+        @instance[:state] = 'Complete'
+      else
+        @instance[:state] = 'Failed'
+      end
     end
 
     @instance.save
@@ -563,7 +575,6 @@ class WhRunPipelineInstance
 
   def cleanup
     if @instance
-      @instance[:active] = false
       @instance.save
     end
   end
@@ -621,9 +632,11 @@ begin
   runner.apply_parameters(p.leftovers)
   runner.setup_instance
   if $options[:submit]
+    runner.instance.set_state 'RunningOnServer'
     runner.instance.save
     puts runner.instance[:uuid]
   else
+    runner.instance.set_state 'RunningOnClient'
     runner.run
   end
 rescue Exception => e
diff --git a/services/api/app/models/pipeline_instance.rb b/services/api/app/models/pipeline_instance.rb
index b16769c..f18fa3a 100644
--- a/services/api/app/models/pipeline_instance.rb
+++ b/services/api/app/models/pipeline_instance.rb
@@ -4,10 +4,13 @@ class PipelineInstance < ArvadosModel
   include CommonApiTemplate
   serialize :components, Hash
   serialize :properties, Hash
+  serialize :components_summary, Hash
   belongs_to :pipeline_template, :foreign_key => :pipeline_template_uuid, :primary_key => :uuid
 
   before_validation :bootstrap_components
   before_validation :update_success
+  before_create :set_state_for_new_pipeline
+  before_save :set_state_for_new_pipeline
 
   api_accessible :user, extend: :common do |t|
     t.add :pipeline_template_uuid
@@ -16,14 +19,89 @@ class PipelineInstance < ArvadosModel
     t.add :components
     t.add :success
     t.add :active
+    t.add :state
     t.add :dependencies
     t.add :properties
+    t.add :components_summary
   end
 
+  # Supported states for a pipeline instance
+  New = 'New'
+  Ready = 'Ready'
+  RunningOnServer = 'RunningOnServer'
+  RunningOnClient = 'RunningOnClient'
+  Paused = 'Paused'
+  Failed = 'Failed'
+  Complete = 'Complete'
+
   def dependencies
     dependency_search(self.components).keys
   end
 
+  def active
+    self.state == RunningOnServer || self.state == RunningOnClient      
+  end
+
+  def success
+    self.state == Complete      
+  end
+
+  def set_state state
+    self.state = state
+  end
+
+  def set_state_for_new_pipeline
+    if !self.state || self.state == New
+      if PipelineInstance.is_ready self.components
+        self.state = Ready
+      else
+        self.state = New
+      end
+    end
+  end
+
+  # if a legacy client tries to update active or success attributes, convert to state
+  def update_attribute name, value
+    if name == 'success'
+      if value == true
+        self.state = Complete
+      else
+        self.state = Failed
+      end
+
+      name = 'state'
+      value = self.state
+    elsif name == 'active'
+      if value == true
+        self.state = RunningOnServer
+      else
+        self.state = New
+      end
+
+      name = 'state'
+      value = self.state
+    end
+
+    super
+  end
+
+  # if all components have input, the pipeline is Ready
+  def self.is_ready components
+    if !components || components.empty?  # is this correct?
+      return true
+    end
+
+    all_components_have_input = true
+    components.each do |name, component|
+      if !component.andand['script_parameters'].andand['input'] || 
+          component.andand['script_parameters'].andand['input'].empty?
+        all_components_have_input = false
+        break
+      end
+    end
+    return all_components_have_input
+  end
+
   def progress_table
     begin
       # v0 pipeline format
@@ -40,7 +118,7 @@ class PipelineInstance < ArvadosModel
         else
           row << 0.0
           if step['failed']
-            self.success = false
+            self.state = Failed
           end
         end
         row << (step['warehousejob']['id'] rescue nil)
@@ -61,7 +139,7 @@ class PipelineInstance < ArvadosModel
   end
 
   def self.queue
-    self.where('active = true')
+    self.where("state = 'Ready' and state != 'RunningOnClient'")
   end
 
   protected
diff --git a/services/api/db/migrate/20140417195429_pipeline_instance_state.rb b/services/api/db/migrate/20140417195429_pipeline_instance_state.rb
new file mode 100644
index 0000000..6f43005
--- /dev/null
+++ b/services/api/db/migrate/20140417195429_pipeline_instance_state.rb
@@ -0,0 +1,45 @@
+class PipelineInstanceState < ActiveRecord::Migration
+  include CurrentApiClient
+
+  def up
+    add_column :pipeline_instances, :state, :string
+    add_column :pipeline_instances, :components_summary, :text
+
+    act_as_system_user do
+      PipelineInstance.all.each do |pi|
+        pi.state = PipelineInstance::New
+puts "\ninstance = #{pi.inspect}"
+puts "\nid = #{pi.id}"
+puts "\nsuccess = #{pi[:success]}"
+puts "\nactive = #{pi[:active]}"
+
+        if !pi.attribute_present? :success   # success is nil
+          if pi[:active] == true
+            pi.state = PipelineInstance::RunningOnServer
+          else
+            if PipelineInstance.is_ready pi.components
+              pi.state = PipelineInstance::Ready
+            else
+              pi.state = PipelineInstance::New
+            end
+          end
+        elsif pi[:success] == true
+          pi.state = PipelineInstance::Complete
+        else
+          pi.state = PipelineInstance::Failed
+        end
+
+        pi.save!
+      end
+    end
+
+    remove_column :pipeline_instances, :active
+    remove_column :pipeline_instances, :success
+  end
+
+  def down
+      add_column :pipeline_instances, :success, :null => true
+      add_column :pipeline_instances, :active, :default => false
+      remove_column :pipeline_instances, :state
+  end
+end
diff --git a/services/api/db/schema.rb b/services/api/db/schema.rb
index e2301e5..cd8f00f 100644
--- a/services/api/db/schema.rb
+++ b/services/api/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20140407184311) do
+ActiveRecord::Schema.define(:version => 20140417195429) do
 
   create_table "api_client_authorizations", :force => true do |t|
     t.string   "api_token",                                           :null => false
@@ -311,10 +311,10 @@ ActiveRecord::Schema.define(:version => 20140407184311) do
     t.string   "pipeline_template_uuid"
     t.string   "name"
     t.text     "components"
-    t.boolean  "success"
-    t.boolean  "active",                  :default => false
     t.datetime "updated_at"
     t.text     "properties"
+    t.string   "state"
+    t.text     "components_summary"
   end
 
   add_index "pipeline_instances", ["created_at"], :name => "index_pipeline_instances_on_created_at"
diff --git a/services/api/test/fixtures/pipeline_instances.yml b/services/api/test/fixtures/pipeline_instances.yml
new file mode 100644
index 0000000..931eb68
--- /dev/null
+++ b/services/api/test/fixtures/pipeline_instances.yml
@@ -0,0 +1,3 @@
+new_pipeline:
+  uuid: zzzzz-xxxxx-f4gneyn6br1xize
+  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
diff --git a/services/api/test/unit/pipeline_instance_test.rb b/services/api/test/unit/pipeline_instance_test.rb
index 9b4c7c3..c6aae4d 100644
--- a/services/api/test/unit/pipeline_instance_test.rb
+++ b/services/api/test/unit/pipeline_instance_test.rb
@@ -1,7 +1,81 @@
 require 'test_helper'
 
 class PipelineInstanceTest < ActiveSupport::TestCase
-  # test "the truth" do
-  #   assert true
-  # end
+
+  test "check active and success for a pipeline in new state" do
+    pi = pipeline_instances :new_pipeline
+
+    assert !pi.active, 'expected active to be false for a new pipeline'
+    assert !pi.success, 'expected success to be false for a new pipeline'
+    assert !pi.state, 'expected state to be nil because the fixture had no state specified'
+
+    # save the pipeline and expect state to be New
+    pi.save
+    pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+    assert_equal PipelineInstance::Ready, pi.state, 'expected state to be New for new pipeline'
+    assert !pi.active, 'expected active to be false for a new pipeline'
+    assert !pi.success, 'expected success to be false for a new pipeline'
+  end
+
+  test "update attributes for pipeline" do
+    pi = pipeline_instances :new_pipeline
+
+    # add a component with no input and expect state to be New
+    component = {'script_parameters' => {"something" => "c1bad4b39ca5a924e481008009d94e32+210"}}
+    pi.components['first'] = component
+    components = pi.components
+    pi.update_attribute 'components', pi.components
+    pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+    assert_equal PipelineInstance::New, pi.state, 'expected state to be Ready after adding component with input'
+    assert_equal pi.components.size, 1, 'expected one component'
+    assert !pi.active, 'expected active to be false after update'
+    assert !pi.success, 'expected success to be false for a new pipeline'
+
+    # add a component with input and expect state to become Ready
+    component = {'script_parameters' => {"input" => "yyyad4b39ca5a924e481008009d94e32+210"}}
+    pi.components['first'] = component
+    components = pi.components
+    pi.update_attribute 'components', pi.components
+    pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+    assert_equal PipelineInstance::Ready, pi.state, 'expected state to be Ready after adding component with input'
+    assert_equal pi.components.size, 1, 'expected one component'
+    assert !pi.active, 'expected active to be false after update'
+    assert !pi.success, 'expected success to be false for a new pipeline'
+   
+    pi.update_attribute 'active', true
+    pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+    assert_equal PipelineInstance::RunningOnServer, pi.state, 'expected state to be RunningOnServer after updating active to true'
+    assert pi.active, 'expected active to be true after update'
+    assert !pi.success, 'expected success to be false for a new pipeline'
+
+    pi.update_attribute 'success', false
+    pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+    assert_equal PipelineInstance::Failed, pi.state, 'expected state to be Failed after updating success to false'
+    assert !pi.active, 'expected active to be false after update'
+    assert !pi.success, 'expected success to be false for a new pipeline'
+
+    pi.update_attribute 'success', true
+    pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+    assert_equal PipelineInstance::Complete, pi.state, 'expected state to be Complete after updating success to true'
+    assert !pi.active, 'expected active to be false after update'
+    assert pi.success, 'expected success to be true after update'
+  end
+
+  test "update attributes for pipeline with two components" do
+    pi = pipeline_instances :new_pipeline
+
+    # add two components, one with input and one with no input and expect state to be New
+    component1 = {'script_parameters' => {"something" => "xxxad4b39ca5a924e481008009d94e32+210", "input" => "c1bad4b39ca5a924e481008009d94e32+210"}}
+    component2 = {'script_parameters' => {"something_else" => "xxxad4b39ca5a924e481008009d94e32+210", "not_input" => "c1bad4b39ca5a924e481008009d94e32+210"}}
+    pi.components['first'] = component1
+    pi.components['second'] = component2
+    components = pi.components
+    pi.update_attribute 'components', pi.components
+    pi = PipelineInstance.find_by_uuid 'zzzzz-xxxxx-f4gneyn6br1xize'
+    assert_equal PipelineInstance::New, pi.state, 'expected state to be Ready after adding component with input'
+    assert_equal pi.components.size, 2, 'expected two components'
+    assert !pi.active, 'expected active to be false after update'
+    assert !pi.success, 'expected success to be false for a new pipeline'
+  end
+
 end

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list