[ARVADOS] updated: 4c8468e142d92a4dd063bde9ea1337b2161999e9

git at public.curoverse.com git at public.curoverse.com
Wed Jan 29 16:57:17 EST 2014


Summary of changes:
 .../app/controllers/collections_controller.rb      |  142 +-------------
 .../controllers/pipeline_instances_controller.rb   |   25 +++
 apps/workbench/app/helpers/provenance_helper.rb    |  219 ++++++++++++++++++++
 apps/workbench/app/views/collections/show.html.erb |    4 +-
 .../app/views/pipeline_instances/show.html.erb     |    3 +
 docker/run.sh                                      |   18 +-
 sdk/cli/Gemfile.lock                               |   14 +-
 sdk/cli/arvados-cli.gemspec                        |   14 +-
 sdk/python/arvados/__init__.py                     |    2 +-
 .../arvados/v1/collections_controller.rb           |   11 +-
 services/api/app/models/job.rb                     |   16 +-
 .../20140117231056_normalize_collection_uuid.rb    |    4 +-
 ...malize_collection_uuids_in_script_parameters.rb |   45 ++++
 services/api/db/schema.rb                          |    2 +-
 14 files changed, 339 insertions(+), 180 deletions(-)
 create mode 100644 apps/workbench/app/helpers/provenance_helper.rb
 create mode 100644 services/api/db/migrate/20140129184311_normalize_collection_uuids_in_script_parameters.rb

       via  4c8468e142d92a4dd063bde9ea1337b2161999e9 (commit)
       via  604d8ddb533176dc2d067189e11376dc8dc81af3 (commit)
       via  3acff4c57e3459dae420903c02c717cd5c07319f (commit)
       via  e3e224295f2a3aa53ee82ebfc3982b90e85cfc08 (commit)
       via  c67624630594b481aa96d548282187720601abea (commit)
       via  e2b7fd4df7f6ba211f2bdbc173ff9cf5de61fbb0 (commit)
       via  53f098a4343081d2f31be9b84c11973320532337 (commit)
       via  31d298b5a3550b02d0f67c6e6110821cd078fbb2 (commit)
       via  3409236dee75142911976ebaef134ae203a726c6 (commit)
       via  2728f59746f7a57ed0283b899f9636cfef77d001 (commit)
       via  566bb36cbadbe0e4ceb1a30123033d9310a0f4d8 (commit)
       via  5f7fa947b13eeee04d56b487f7173308635aa60c (commit)
       via  ca5197de7e459a48d72cda8e1f4de4c4201366fb (commit)
       via  c106ef5033c4fcf19e6409fa517030ecffb3e2ea (commit)
       via  68a05d89a28f609e3592d241bf6d0417c75d219a (commit)
      from  04bb94072486e32ed9784be3a22c43d961c84bee (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 4c8468e142d92a4dd063bde9ea1337b2161999e9
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Jan 29 16:58:03 2014 -0500

    Now renders collections using up to the first three files in the collection

diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
index de3401d..dbc7dbb 100644
--- a/apps/workbench/app/helpers/provenance_helper.rb
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -11,17 +11,26 @@ module ProvenanceHelper
           if pdata[uuid][:name]
             return "\"#{uuid}\" [label=\"#{pdata[uuid][:name]}\",href=\"#{href}\",shape=oval];\n"
           else
-            i = 0
-            label = ""
-            while i < 3 and i < pdata[uuid].files.length
-              label += "\\n" unless label == ""
-              label += pdata[uuid].files[i][1]
-              i += 1
+            files = nil
+            if pdata[uuid].respond_to? :files
+              files = pdata[uuid].files
+            elsif pdata[uuid][:files]
+              files = pdata[uuid][:files]
             end
-            if i < pdata[uuid].files.length
-              label += "\\n⋮"
+            
+            if files
+              i = 0
+              label = ""
+              while i < 3 and i < files.length
+                label += "\\n" unless label == ""
+                label += files[i][1]
+                i += 1
+              end
+              if i < files.length
+                label += "\\n⋮"
+              end
+              return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval];\n"
             end
-            return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval];\n"
           end  
         end
         return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\"];\n"
@@ -88,9 +97,9 @@ module ProvenanceHelper
         unless node == ""
           node += "']"
           #puts node
-          id = "#{job[:uuid]}_#{prefix}"
-          gr += "\"#{id}\" [label=\"#{node}\"];\n"
-          gr += edge(job_uuid(job), id, {:label => prefix}, opts)        
+          #id = "#{job[:uuid]}_#{prefix}"
+          gr += "\"#{node}\" [label=\"#{node}\"];\n"
+          gr += edge(job_uuid(job), node, {:label => prefix}, opts)        
         end
       else
         m = collection_uuid(sp)
@@ -98,9 +107,9 @@ module ProvenanceHelper
           gr += edge(job_uuid(job), m, {:label => prefix}, opts)
           gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, m, opts)
         elsif opts[:all_script_parameters]
-          id = "#{job[:uuid]}_#{prefix}"
-          gr += "\"#{id}\" [label=\"#{sp}\"];\n"
-          gr += edge(job_uuid(job), id, {:label => prefix}, opts)
+          #id = "#{job[:uuid]}_#{prefix}"
+          gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
+          gr += edge(job_uuid(job), sp, {:label => prefix}, opts)
         end
       end
     end
diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index ad2d6d1..9198f58 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -93,6 +93,10 @@ class Arvados::V1::CollectionsController < ApplicationController
       # uuid is a collection
       Collection.where(uuid: uuid).each do |c|
         visited[uuid] = c.as_api_response
+        visited[uuid][:files] = []
+        c.files.each do |f|
+          visited[uuid][:files] << f
+        end
       end
 
       Job.where(output: uuid).each do |job|

commit 604d8ddb533176dc2d067189e11376dc8dc81af3
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Jan 29 16:36:53 2014 -0500

    Very nice provenance diagram for pipelines.

diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index ca176b4..27f5bb7 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -99,7 +99,7 @@ class CollectionsController < ApplicationController
     end
     
     Collection.where(uuid: @object.uuid).each do |u|
-      @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, u.uuid
+      @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, u.uuid, {:direction => :bottom_up}
     end
   end
 
diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb
index 79aade7..37e743f 100644
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@ -8,19 +8,19 @@ class PipelineInstancesController < ApplicationController
       j = v[:job]
       somejob = j[:uuid]
       provenance[somejob.intern] = j
-      collections << j[:output]
+      collections << j[:output].intern
+      j[:dependencies].each do |k|
+        collections << k.intern
+      end
     end
 
-    puts collections
-    puts '---'
-
     Collection.where(uuid: collections).each do |c|
-      puts c.uuid
+      #puts c.uuid
       provenance[c.uuid.intern] = c
     end
 
     PipelineInstance.where(uuid: @object.uuid).each do |u|
-      @prov_svg = CollectionsController::create_provenance_graph provenance, somejob
+      @prov_svg = ProvenanceHelper::create_provenance_graph provenance, collections, {:all_script_parameters => true}
     end
   end
 
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
index c393a1b..de3401d 100644
--- a/apps/workbench/app/helpers/provenance_helper.rb
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -1,12 +1,33 @@
-class ProvenanceHelper
-  def self.describe_node(uuid)
-    uuid = uuid.to_s
-    rsc = ArvadosBase::resource_class_for_uuid uuid
+module ProvenanceHelper
+  def self.describe_node(pdata, uuid)
+    rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
     if rsc
-      "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
-    else
-      ""
+      href = "/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}"
+
+      #"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
+      if rsc == Collection
+        if pdata[uuid] 
+          #puts pdata[uuid]
+          if pdata[uuid][:name]
+            return "\"#{uuid}\" [label=\"#{pdata[uuid][:name]}\",href=\"#{href}\",shape=oval];\n"
+          else
+            i = 0
+            label = ""
+            while i < 3 and i < pdata[uuid].files.length
+              label += "\\n" unless label == ""
+              label += pdata[uuid].files[i][1]
+              i += 1
+            end
+            if i < pdata[uuid].files.length
+              label += "\\n⋮"
+            end
+            return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval];\n"
+          end  
+        end
+        return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\"];\n"
+      end
     end
+    return ""
   end
 
   def self.job_uuid(job)
@@ -23,7 +44,24 @@ class ProvenanceHelper
     end
   end
 
-  def self.script_param_edges(visited, job, prefix, sp, opts)
+  def self.edge(tail, head, extra, opts)
+    if opts[:direction] == :bottom_up
+      gr = "\"#{tail}\" -> \"#{head}\""
+    else
+      gr = "\"#{head}\" -> \"#{tail}\""
+    end
+    if extra.length > 0
+      gr += "["
+      extra.each do |k, v|
+        gr += "#{k}=\"#{v}\","
+      end
+      gr += "]"
+    end
+    gr += ";\n"
+    gr
+  end
+
+  def self.script_param_edges(pdata, visited, job, prefix, sp, opts)
     gr = ""
     if sp and not sp.empty?
       case sp
@@ -32,57 +70,76 @@ class ProvenanceHelper
           if prefix.size > 0
             k = prefix + "::" + k.to_s
           end
-          gr += CollectionsController::script_param_edges(visited, job, k.to_s, v, opts)
+          gr += ProvenanceHelper::script_param_edges(pdata, visited, job, k.to_s, v, opts)
         end
       when Array
+        i = 0
+        node = ""
         sp.each do |v|
-          gr += CollectionsController::script_param_edges(visited, job, prefix, v, opts)
+          if collection_uuid(v)
+            gr += ProvenanceHelper::script_param_edges(pdata, visited, job, "#{prefix}[#{i}]", v, opts)
+          else
+            node += "', '" unless node == ""
+            node = "['" if node == ""
+            node += "#{v}"
+          end
+          i += 1
+        end
+        unless node == ""
+          node += "']"
+          #puts node
+          id = "#{job[:uuid]}_#{prefix}"
+          gr += "\"#{id}\" [label=\"#{node}\"];\n"
+          gr += edge(job_uuid(job), id, {:label => prefix}, opts)        
         end
       else
         m = collection_uuid(sp)
         if m
-          gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
-          gr += CollectionsController::generate_provenance_edges(visited, m, opts)
+          gr += edge(job_uuid(job), m, {:label => prefix}, opts)
+          gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, m, opts)
+        elsif opts[:all_script_parameters]
+          id = "#{job[:uuid]}_#{prefix}"
+          gr += "\"#{id}\" [label=\"#{sp}\"];\n"
+          gr += edge(job_uuid(job), id, {:label => prefix}, opts)
         end
       end
     end
     gr
   end
 
-  def self.generate_provenance_edges(pdata, uuid, opts)
+  def self.generate_provenance_edges(pdata, visited, uuid, opts)
     gr = ""
-    m = CollectionsController::collection_uuid(uuid)
+    m = ProvenanceHelper::collection_uuid(uuid)
     uuid = m if m
 
     uuid = uuid.intern if uuid
 
-    if (not uuid) or uuid.empty? \
-      or (pdata[uuid] and pdata[uuid][:_visited])
+    if (not uuid) or uuid.empty? or visited[uuid]
 
       #puts "already visited #{uuid}"
       return ""
     end
 
     if not pdata[uuid] then 
-      return CollectionsController::describe_node(uuid) 
+      return ProvenanceHelper::describe_node(pdata, uuid)
     else
-      pdata[uuid][:_visited] = true
+      visited[uuid] = true
     end
 
     #puts "visiting #{uuid}"
 
     if m  
       # uuid is a collection
-      gr += CollectionsController::describe_node(uuid)
+      gr += ProvenanceHelper::describe_node(pdata, uuid)
 
       pdata.each do |k, job|
         if job[:output] == uuid.to_s
-          gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"output\"];"
-          gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
+          gr += self.edge(uuid, job_uuid(job), {:label => "output"}, opts)
+          gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, job[:uuid], opts)
         end
         if job[:log] == uuid.to_s
-          gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"log\"];"
-          gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
+          gr += edge(uuid, job_uuid(job), {:label => "log"}, opts)
+          gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, job[:uuid], opts)
         end
       end
     else
@@ -92,18 +149,18 @@ class ProvenanceHelper
       if rsc == Job
         job = pdata[uuid]
         if job
-          gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters], opts)
+          gr += ProvenanceHelper::script_param_edges(pdata, visited, job, "", job[:script_parameters], opts)
         end
       else
-        gr += CollectionsController::describe_node(uuid)
+        gr += ProvenanceHelper::describe_node(pdata, uuid)
       end
     end
 
     pdata.each do |k, link|
       if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
-        gr += CollectionsController::describe_node(link[:tail_uuid])
-        gr += "\"#{link[:head_uuid]}\" -> \"#{link[:tail_uuid]}\" [label=\" #{link[:name]}\", href=\"/links/#{link[:uuid]}\"];"
-        gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid], opts)
+        gr += ProvenanceHelper::describe_node(pdata, link[:tail_uuid])
+        gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => "/links/#{link[:uuid]}"}, opts) 
+        gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, link[:tail_uuid], opts)
       end
     end
 
@@ -117,15 +174,28 @@ class ProvenanceHelper
     
     gr = """strict digraph {
 node [fontsize=8,shape=box];
-edge [dir=back,fontsize=8];"""
+edge [fontsize=8];"""
+
+    if opts[:direction] == :bottom_up
+      gr += "edge [dir=back];"
+    end
 
     #puts "pdata is #{pdata}"
 
-    gr += CollectionsController::generate_provenance_edges(pdata, uuid, opts)
+    visited = {}
+    if uuid.respond_to? :each
+      uuid.each do |u|
+        gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, u, opts)
+      end
+    else
+      gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, uuid, opts)
+    end
 
     gr += "}"
     svg = ""
 
+    #puts gr
+
     Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
       stdin.print(gr)
       stdin.close

commit 3acff4c57e3459dae420903c02c717cd5c07319f
Merge: e3e2242 c676246
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Jan 29 14:58:53 2014 -0500

    Merge remote-tracking branch 'origin/master' into 1977-provenance-report


commit e3e224295f2a3aa53ee82ebfc3982b90e85cfc08
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Jan 29 14:58:11 2014 -0500

    Started refactoring

diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 3a1013a..ca176b4 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -52,144 +52,6 @@ class CollectionsController < ApplicationController
     self.response_body = FileStreamer.new opts
   end
 
-  def self.describe_node(uuid)
-    uuid = uuid.to_s
-    rsc = ArvadosBase::resource_class_for_uuid uuid
-    if rsc
-      "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
-    else
-      ""
-    end
-  end
-
-  def self.job_uuid(job)
-    # "#{job[:script]}\\n#{job[:script_version]}"
-    "#{job[:script]}"
-  end
-
-  def self.collection_uuid(uuid)
-    m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid.to_s)
-    if m
-      m[1]
-    else
-      nil
-    end
-  end
-
-  def self.script_param_edges(visited, job, prefix, sp, opts)
-    gr = ""
-    if sp and not sp.empty?
-      case sp
-      when Hash
-        sp.each do |k, v|
-          if prefix.size > 0
-            k = prefix + "::" + k.to_s
-          end
-          gr += CollectionsController::script_param_edges(visited, job, k.to_s, v, opts)
-        end
-      when Array
-        sp.each do |v|
-          gr += CollectionsController::script_param_edges(visited, job, prefix, v, opts)
-        end
-      else
-        m = collection_uuid(sp)
-        if m
-          gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
-          gr += CollectionsController::generate_provenance_edges(visited, m, opts)
-        end
-      end
-    end
-    gr
-  end
-
-  def self.generate_provenance_edges(pdata, uuid, opts)
-    gr = ""
-    m = CollectionsController::collection_uuid(uuid)
-    uuid = m if m
-
-    uuid = uuid.intern if uuid
-
-    if (not uuid) or uuid.empty? \
-      or (pdata[uuid] and pdata[uuid][:_visited])
-
-      #puts "already visited #{uuid}"
-      return ""
-    end
-
-    if not pdata[uuid] then 
-      return CollectionsController::describe_node(uuid) 
-    else
-      pdata[uuid][:_visited] = true
-    end
-
-    #puts "visiting #{uuid}"
-
-    if m  
-      # uuid is a collection
-      gr += CollectionsController::describe_node(uuid)
-
-      pdata.each do |k, job|
-        if job[:output] == uuid.to_s
-          gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"output\"];"
-          gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
-        end
-        if job[:log] == uuid.to_s
-          gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"log\"];"
-          gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
-        end
-      end
-    else
-      # uuid is something else
-      rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
-
-      if rsc == Job
-        job = pdata[uuid]
-        if job
-          gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters], opts)
-        end
-      else
-        gr += CollectionsController::describe_node(uuid)
-      end
-    end
-
-    pdata.each do |k, link|
-      if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
-        gr += CollectionsController::describe_node(link[:tail_uuid])
-        gr += "\"#{link[:head_uuid]}\" -> \"#{link[:tail_uuid]}\" [label=\" #{link[:name]}\", href=\"/links/#{link[:uuid]}\"];"
-        gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid], opts)
-      end
-    end
-
-    #puts "finished #{uuid}"
-
-    gr
-  end
-
-  def self.create_provenance_graph(pdata, uuid, opts={})
-    require 'open3'
-    
-    gr = """strict digraph {
-node [fontsize=8,shape=box];
-edge [dir=back,fontsize=8];"""
-
-    #puts "pdata is #{pdata}"
-
-    gr += CollectionsController::generate_provenance_edges(pdata, uuid, opts)
-
-    gr += "}"
-    svg = ""
-
-    Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
-      stdin.print(gr)
-      stdin.close
-      svg = stdout.read()
-      wait_thr.value
-      stdout.close()
-    end
-
-    svg = svg.sub(/<\?xml.*?\?>/m, "")
-    svg = svg.sub(/<!DOCTYPE.*?>/m, "")
-  end
 
   def show
     return super if !@object
@@ -237,7 +99,7 @@ edge [dir=back,fontsize=8];"""
     end
     
     Collection.where(uuid: @object.uuid).each do |u|
-      @prov_svg = CollectionsController::create_provenance_graph u.provenance, u.uuid
+      @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, u.uuid
     end
   end
 
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
new file mode 100644
index 0000000..c393a1b
--- /dev/null
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -0,0 +1,140 @@
+class ProvenanceHelper
+  def self.describe_node(uuid)
+    uuid = uuid.to_s
+    rsc = ArvadosBase::resource_class_for_uuid uuid
+    if rsc
+      "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
+    else
+      ""
+    end
+  end
+
+  def self.job_uuid(job)
+    # "#{job[:script]}\\n#{job[:script_version]}"
+    "#{job[:script]}"
+  end
+
+  def self.collection_uuid(uuid)
+    m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid.to_s)
+    if m
+      m[1]
+    else
+      nil
+    end
+  end
+
+  def self.script_param_edges(visited, job, prefix, sp, opts)
+    gr = ""
+    if sp and not sp.empty?
+      case sp
+      when Hash
+        sp.each do |k, v|
+          if prefix.size > 0
+            k = prefix + "::" + k.to_s
+          end
+          gr += CollectionsController::script_param_edges(visited, job, k.to_s, v, opts)
+        end
+      when Array
+        sp.each do |v|
+          gr += CollectionsController::script_param_edges(visited, job, prefix, v, opts)
+        end
+      else
+        m = collection_uuid(sp)
+        if m
+          gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
+          gr += CollectionsController::generate_provenance_edges(visited, m, opts)
+        end
+      end
+    end
+    gr
+  end
+
+  def self.generate_provenance_edges(pdata, uuid, opts)
+    gr = ""
+    m = CollectionsController::collection_uuid(uuid)
+    uuid = m if m
+
+    uuid = uuid.intern if uuid
+
+    if (not uuid) or uuid.empty? \
+      or (pdata[uuid] and pdata[uuid][:_visited])
+
+      #puts "already visited #{uuid}"
+      return ""
+    end
+
+    if not pdata[uuid] then 
+      return CollectionsController::describe_node(uuid) 
+    else
+      pdata[uuid][:_visited] = true
+    end
+
+    #puts "visiting #{uuid}"
+
+    if m  
+      # uuid is a collection
+      gr += CollectionsController::describe_node(uuid)
+
+      pdata.each do |k, job|
+        if job[:output] == uuid.to_s
+          gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"output\"];"
+          gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
+        end
+        if job[:log] == uuid.to_s
+          gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"log\"];"
+          gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
+        end
+      end
+    else
+      # uuid is something else
+      rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
+
+      if rsc == Job
+        job = pdata[uuid]
+        if job
+          gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters], opts)
+        end
+      else
+        gr += CollectionsController::describe_node(uuid)
+      end
+    end
+
+    pdata.each do |k, link|
+      if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
+        gr += CollectionsController::describe_node(link[:tail_uuid])
+        gr += "\"#{link[:head_uuid]}\" -> \"#{link[:tail_uuid]}\" [label=\" #{link[:name]}\", href=\"/links/#{link[:uuid]}\"];"
+        gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid], opts)
+      end
+    end
+
+    #puts "finished #{uuid}"
+
+    gr
+  end
+
+  def self.create_provenance_graph(pdata, uuid, opts={})
+    require 'open3'
+    
+    gr = """strict digraph {
+node [fontsize=8,shape=box];
+edge [dir=back,fontsize=8];"""
+
+    #puts "pdata is #{pdata}"
+
+    gr += CollectionsController::generate_provenance_edges(pdata, uuid, opts)
+
+    gr += "}"
+    svg = ""
+
+    Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
+      stdin.print(gr)
+      stdin.close
+      svg = stdout.read()
+      wait_thr.value
+      stdout.close()
+    end
+
+    svg = svg.sub(/<\?xml.*?\?>/m, "")
+    svg = svg.sub(/<!DOCTYPE.*?>/m, "")
+  end
+end

commit e2b7fd4df7f6ba211f2bdbc173ff9cf5de61fbb0
Merge: 31d298b 53f098a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Jan 29 14:41:55 2014 -0500

    Merge branch 'master' into 1977-provenance-report


commit 31d298b5a3550b02d0f67c6e6110821cd078fbb2
Merge: 3409236 2728f59
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Jan 29 14:15:25 2014 -0500

    Merge commit '2728f59' into 1977-provenance-report


commit 3409236dee75142911976ebaef134ae203a726c6
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Jan 29 14:14:36 2014 -0500

    Working on visualization of pipelines

diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index dcf1d91..3a1013a 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -58,7 +58,6 @@ class CollectionsController < ApplicationController
     if rsc
       "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
     else
-      puts "error! #{uuid} #{rsc}"
       ""
     end
   end
@@ -77,7 +76,7 @@ class CollectionsController < ApplicationController
     end
   end
 
-  def self.script_param_edges(visited, job, prefix, sp)
+  def self.script_param_edges(visited, job, prefix, sp, opts)
     gr = ""
     if sp and not sp.empty?
       case sp
@@ -86,24 +85,24 @@ class CollectionsController < ApplicationController
           if prefix.size > 0
             k = prefix + "::" + k.to_s
           end
-          gr += CollectionsController::script_param_edges(visited, job, k.to_s, v)
+          gr += CollectionsController::script_param_edges(visited, job, k.to_s, v, opts)
         end
       when Array
         sp.each do |v|
-          gr += CollectionsController::script_param_edges(visited, job, prefix, v)
+          gr += CollectionsController::script_param_edges(visited, job, prefix, v, opts)
         end
       else
         m = collection_uuid(sp)
         if m
           gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
-          gr += CollectionsController::generate_provenance_edges(visited, m)
+          gr += CollectionsController::generate_provenance_edges(visited, m, opts)
         end
       end
     end
     gr
   end
 
-  def self.generate_provenance_edges(pdata, uuid)
+  def self.generate_provenance_edges(pdata, uuid, opts)
     gr = ""
     m = CollectionsController::collection_uuid(uuid)
     uuid = m if m
@@ -113,7 +112,7 @@ class CollectionsController < ApplicationController
     if (not uuid) or uuid.empty? \
       or (pdata[uuid] and pdata[uuid][:_visited])
 
-      puts "already visited #{uuid}"
+      #puts "already visited #{uuid}"
       return ""
     end
 
@@ -123,7 +122,7 @@ class CollectionsController < ApplicationController
       pdata[uuid][:_visited] = true
     end
 
-    puts "visiting #{uuid}"
+    #puts "visiting #{uuid}"
 
     if m  
       # uuid is a collection
@@ -146,7 +145,7 @@ class CollectionsController < ApplicationController
       if rsc == Job
         job = pdata[uuid]
         if job
-          gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters])
+          gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters], opts)
         end
       else
         gr += CollectionsController::describe_node(uuid)
@@ -157,25 +156,25 @@ class CollectionsController < ApplicationController
       if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
         gr += CollectionsController::describe_node(link[:tail_uuid])
         gr += "\"#{link[:head_uuid]}\" -> \"#{link[:tail_uuid]}\" [label=\" #{link[:name]}\", href=\"/links/#{link[:uuid]}\"];"
-        gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid])
+        gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid], opts)
       end
     end
 
-    puts "finished #{uuid}"
+    #puts "finished #{uuid}"
 
     gr
   end
 
-  def self.create_provenance_graph(pdata, uuid)
+  def self.create_provenance_graph(pdata, uuid, opts={})
     require 'open3'
     
     gr = """strict digraph {
 node [fontsize=8,shape=box];
 edge [dir=back,fontsize=8];"""
 
-    puts "pdata is #{pdata}"
+    #puts "pdata is #{pdata}"
 
-    gr += CollectionsController::generate_provenance_edges(pdata, uuid)
+    gr += CollectionsController::generate_provenance_edges(pdata, uuid, opts)
 
     gr += "}"
     svg = ""
diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb
index afb9949..79aade7 100644
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@ -1,2 +1,27 @@
 class PipelineInstancesController < ApplicationController
+
+  def show
+    provenance = {}
+    somejob = nil
+    collections = []
+    @object.components.each do |k, v|
+      j = v[:job]
+      somejob = j[:uuid]
+      provenance[somejob.intern] = j
+      collections << j[:output]
+    end
+
+    puts collections
+    puts '---'
+
+    Collection.where(uuid: collections).each do |c|
+      puts c.uuid
+      provenance[c.uuid.intern] = c
+    end
+
+    PipelineInstance.where(uuid: @object.uuid).each do |u|
+      @prov_svg = CollectionsController::create_provenance_graph provenance, somejob
+    end
+  end
+
 end
diff --git a/apps/workbench/app/views/pipeline_instances/show.html.erb b/apps/workbench/app/views/pipeline_instances/show.html.erb
index 74c090d..2745e76 100644
--- a/apps/workbench/app/views/pipeline_instances/show.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/show.html.erb
@@ -28,6 +28,9 @@
     <% end %>
   </tbody>
 </table>
+
+<%= raw(@prov_svg) %>
+
 <pre>
 <%= JSON.pretty_generate(@object.attributes) rescue nil %>
 </pre>

commit 5f7fa947b13eeee04d56b487f7173308635aa60c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Wed Jan 29 11:38:27 2014 -0500

    Fixed bugs in creating graph from API provenance report.

diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 57b9bab..dcf1d91 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -58,18 +58,14 @@ class CollectionsController < ApplicationController
     if rsc
       "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
     else
+      puts "error! #{uuid} #{rsc}"
       ""
     end
   end
 
-  def self.describe_script(job)
-    # """\"#{job.script_version}\" [label=\"#{job.script}: #{job.script_version}\"];
-    #   \"#{job.uuid}\" -> \"#{job.script_version}\" [label=\"script\"];"""
-    "\"#{job.uuid}\" [label=\"#{job.script}\\n#{job.script_version}\"];"
-  end
-
   def self.job_uuid(job)
-    "#{job.script}\\n#{job.script_version}"
+    # "#{job[:script]}\\n#{job[:script_version]}"
+    "#{job[:script]}"
   end
 
   def self.collection_uuid(uuid)
@@ -115,39 +111,42 @@ class CollectionsController < ApplicationController
     uuid = uuid.intern if uuid
 
     if (not uuid) or uuid.empty? \
-      or (pdata[uuid] and pdata[uuid][:_visited]) \
-      or (not pdata[uuid])
+      or (pdata[uuid] and pdata[uuid][:_visited])
 
       puts "already visited #{uuid}"
       return ""
     end
 
-    puts "visiting #{uuid}"
+    if not pdata[uuid] then 
+      return CollectionsController::describe_node(uuid) 
+    else
+      pdata[uuid][:_visited] = true
+    end
 
-    pdata[uuid][:_visited] = true
+    puts "visiting #{uuid}"
 
     if m  
       # uuid is a collection
       gr += CollectionsController::describe_node(uuid)
 
       pdata.each do |k, job|
-        if job[:output] == uuid
+        if job[:output] == uuid.to_s
           gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"output\"];"
           gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
         end
-        if job[:log] == uuid
+        if job[:log] == uuid.to_s
           gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"log\"];"
           gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
         end
       end
     else
       # uuid is something else
-      rsc = ArvadosBase::resource_class_for_uuid uuid
+      rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
 
       if rsc == Job
         job = pdata[uuid]
         if job
-          gr += CollectionsController::script_param_edges(pdata, job, "", job.script_parameters)
+          gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters])
         end
       else
         gr += CollectionsController::describe_node(uuid)
diff --git a/apps/workbench/app/views/collections/show.html.erb b/apps/workbench/app/views/collections/show.html.erb
index adb8bf0..8c6e83e 100644
--- a/apps/workbench/app/views/collections/show.html.erb
+++ b/apps/workbench/app/views/collections/show.html.erb
@@ -210,8 +210,8 @@
 	  </td><td>
 	    <%= j.script_version %>
 	  </td><td>
-            <span class="label <%= if j.success then 'label-success'; elsif j.active then 'label-primary'; else 'label-warning'; end %>">
-	      <%= j.success || j.active ? 'ok' : 'failed' %>
+            <span class="label <%= if j.success then 'label-success'; elsif j.running then 'label-primary'; else 'label-warning'; end %>">
+	      <%= j.success || j.running ? 'ok' : 'failed' %>
             </span>
 	  </td><td>
 	    <%= j.started_at %>
diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 48544a2..ad2d6d1 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -77,13 +77,13 @@ class Arvados::V1::CollectionsController < ApplicationController
         end
       end
     end
-    gr
   end
 
   def generate_provenance_edges(visited, uuid)
     m = collection_uuid(uuid)
+    uuid = m if m
 
-    if not uuid or uuid.empty? or visited[uuid] or visited[m]
+    if not uuid or uuid.empty? or visited[uuid]
       return ""
     end
 
@@ -91,7 +91,6 @@ class Arvados::V1::CollectionsController < ApplicationController
 
     if m  
       # uuid is a collection
-      uuid = m
       Collection.where(uuid: uuid).each do |c|
         visited[uuid] = c.as_api_response
       end
@@ -110,7 +109,7 @@ class Arvados::V1::CollectionsController < ApplicationController
       if rsc == Job
         Job.where(uuid: uuid).each do |job|
           visited[uuid] = job.as_api_response
-          script_param_edges(visited, job, "", job.script_parameters)
+          script_param_edges(visited, job.script_parameters)
         end
       elsif rsc != nil
         rsc.where(uuid: uuid).each do |r|

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list