[ARVADOS] updated: aa66a90eecda1fc79f869ceee36eaed12d806145
git at public.curoverse.com
git at public.curoverse.com
Thu Jan 30 16:18:27 EST 2014
Summary of changes:
.../app/controllers/collections_controller.rb | 2 +-
apps/workbench/app/controllers/jobs_controller.rb | 21 +-
.../controllers/pipeline_instances_controller.rb | 4 +-
apps/workbench/app/helpers/provenance_helper.rb | 384 ++++++++++++--------
apps/workbench/app/views/jobs/index.html.erb | 2 +
5 files changed, 256 insertions(+), 157 deletions(-)
via aa66a90eecda1fc79f869ceee36eaed12d806145 (commit)
from 4c8468e142d92a4dd063bde9ea1337b2161999e9 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit aa66a90eecda1fc79f869ceee36eaed12d806145
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jan 30 16:18:43 2014 -0500
Further improvements to provenance graphs, added ability to select a few jobs and shows the graph for those jobs.
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 27f5bb7..caf49ba 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -99,7 +99,7 @@ class CollectionsController < ApplicationController
end
Collection.where(uuid: @object.uuid).each do |u|
- @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, u.uuid, {:direction => :bottom_up}
+ @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, {:direction => :bottom_up, :combine_jobs => true}
end
end
diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index 1cd3c3e..061db6d 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -1,5 +1,24 @@
class JobsController < ApplicationController
def index
- @jobs = Job.all
+ @svg = ""
+ if params[:uuid]
+ @jobs = Job.where(uuid: params[:uuid])
+ nodes = []
+ collections = []
+ @jobs.each do |j|
+ nodes << j
+ collections << j[:output]
+ collections.concat(ProvenanceHelper::find_collections(j[:script_parameters]))
+ nodes << {:uuid => j[:script_version]}
+ end
+
+ Collection.where(uuid: collections).each do |c|
+ nodes << c
+ end
+
+ @svg = ProvenanceHelper::create_provenance_graph(nodes, {:all_script_parameters => true, :script_version_nodes => true})
+ else
+ @jobs = Job.all
+ end
end
end
diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb
index 37e743f..a926940 100644
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@ -19,9 +19,7 @@ class PipelineInstancesController < ApplicationController
provenance[c.uuid.intern] = c
end
- PipelineInstance.where(uuid: @object.uuid).each do |u|
- @prov_svg = ProvenanceHelper::create_provenance_graph provenance, collections, {:all_script_parameters => true}
- end
+ @prov_svg = ProvenanceHelper::create_provenance_graph provenance, {:all_script_parameters => true}
end
end
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
index dbc7dbb..9d9f44b 100644
--- a/apps/workbench/app/helpers/provenance_helper.rb
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -1,210 +1,268 @@
module ProvenanceHelper
- def self.describe_node(pdata, uuid)
- rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
- if rsc
- href = "/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}"
-
- #"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
- if rsc == Collection
- if pdata[uuid]
- #puts pdata[uuid]
- if pdata[uuid][:name]
- return "\"#{uuid}\" [label=\"#{pdata[uuid][:name]}\",href=\"#{href}\",shape=oval];\n"
- else
- files = nil
- if pdata[uuid].respond_to? :files
- files = pdata[uuid].files
- elsif pdata[uuid][:files]
- files = pdata[uuid][:files]
- end
-
- if files
- i = 0
- label = ""
- while i < 3 and i < files.length
- label += "\\n" unless label == ""
- label += files[i][1]
- i += 1
+
+ class GenerateGraph
+ def initialize(pdata, opts)
+ @pdata = pdata
+ @opts = opts
+ @visited = {}
+ @jobs = {}
+ end
+
+ def self.collection_uuid(uuid)
+ m = /^([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?$/.match(uuid.to_s)
+ if m
+ #if m[2]
+ return m[1]
+ #else
+ # Collection.where(uuid: ['contains', m[1]]).each do |u|
+ # puts "fixup #{uuid} to #{u.uuid}"
+ # return u.uuid
+ # end
+ #end
+ else
+ nil
+ end
+ end
+
+ def describe_node(uuid)
+ rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
+ if rsc
+ href = "/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}"
+
+ #"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
+ if rsc == Collection
+ if @pdata[uuid]
+ #puts @pdata[uuid]
+ if @pdata[uuid][:name]
+ return "\"#{uuid}\" [label=\"#{@pdata[uuid][:name]}\",href=\"#{href}\",shape=oval];\n"
+ else
+ files = nil
+ if @pdata[uuid].respond_to? :files
+ files = @pdata[uuid].files
+ elsif @pdata[uuid][:files]
+ files = @pdata[uuid][:files]
end
- if i < files.length
- label += "\\n⋮"
+
+ if files
+ i = 0
+ label = ""
+ while i < 3 and i < files.length
+ label += "\\n" unless label == ""
+ label += files[i][1]
+ i += 1
+ end
+ if i < files.length
+ label += "\\n⋮"
+ end
+ return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval];\n"
end
- return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval];\n"
- end
- end
+ end
+ end
+ return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\"];\n"
end
- return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\"];\n"
end
+ return ""
end
- return ""
- end
- def self.job_uuid(job)
- # "#{job[:script]}\\n#{job[:script_version]}"
- "#{job[:script]}"
- end
+ def job_uuid(job)
+ if @opts[:combine_jobs]
+ uuid = "#{job[:script]}"
+ else
+ uuid = "#{job[:uuid]}"
+ end
- def self.collection_uuid(uuid)
- m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid.to_s)
- if m
- m[1]
- else
- nil
- end
- end
+ @jobs[uuid] = [] unless @jobs[uuid]
+ @jobs[uuid] << job unless @jobs[uuid].include? job
- def self.edge(tail, head, extra, opts)
- if opts[:direction] == :bottom_up
- gr = "\"#{tail}\" -> \"#{head}\""
- else
- gr = "\"#{head}\" -> \"#{tail}\""
+ uuid
end
- if extra.length > 0
- gr += "["
- extra.each do |k, v|
- gr += "#{k}=\"#{v}\","
+
+ def edge(tail, head, extra)
+ if @opts[:direction] == :bottom_up
+ gr = "\"#{tail}\" -> \"#{head}\""
+ else
+ gr = "\"#{head}\" -> \"#{tail}\""
end
- gr += "]"
+ if extra.length > 0
+ gr += "["
+ extra.each do |k, v|
+ gr += "#{k}=\"#{v}\","
+ end
+ gr += "]"
+ end
+ gr += ";\n"
+ gr
end
- gr += ";\n"
- gr
- end
- def self.script_param_edges(pdata, visited, job, prefix, sp, opts)
- gr = ""
- if sp and not sp.empty?
- case sp
- when Hash
- sp.each do |k, v|
- if prefix.size > 0
- k = prefix + "::" + k.to_s
+ def script_param_edges(job, prefix, sp)
+ gr = ""
+ if sp and not sp.empty?
+ case sp
+ when Hash
+ sp.each do |k, v|
+ if prefix.size > 0
+ k = prefix + "::" + k.to_s
+ end
+ gr += script_param_edges(job, k.to_s, v)
end
- gr += ProvenanceHelper::script_param_edges(pdata, visited, job, k.to_s, v, opts)
- end
- when Array
- i = 0
- node = ""
- sp.each do |v|
- if collection_uuid(v)
- gr += ProvenanceHelper::script_param_edges(pdata, visited, job, "#{prefix}[#{i}]", v, opts)
- else
- node += "', '" unless node == ""
- node = "['" if node == ""
- node += "#{v}"
+ when Array
+ i = 0
+ node = ""
+ sp.each do |v|
+ if GenerateGraph::collection_uuid(v)
+ gr += script_param_edges(job, "#{prefix}[#{i}]", v)
+ else
+ node += "', '" unless node == ""
+ node = "['" if node == ""
+ node += "#{v}"
+ end
+ i += 1
+ end
+ unless node == ""
+ node += "']"
+ #puts node
+ #id = "#{job[:uuid]}_#{prefix}"
+ gr += "\"#{node}\" [label=\"#{node}\"];\n"
+ gr += edge(job_uuid(job), node, {:label => prefix})
+ end
+ else
+ m = GenerateGraph::collection_uuid(sp)
+ if m
+ gr += edge(job_uuid(job), m, {:label => prefix})
+ gr += generate_provenance_edges(m)
+ elsif @opts[:all_script_parameters]
+ #id = "#{job[:uuid]}_#{prefix}"
+ gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
+ gr += edge(job_uuid(job), sp, {:label => prefix})
end
- i += 1
- end
- unless node == ""
- node += "']"
- #puts node
- #id = "#{job[:uuid]}_#{prefix}"
- gr += "\"#{node}\" [label=\"#{node}\"];\n"
- gr += edge(job_uuid(job), node, {:label => prefix}, opts)
- end
- else
- m = collection_uuid(sp)
- if m
- gr += edge(job_uuid(job), m, {:label => prefix}, opts)
- gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, m, opts)
- elsif opts[:all_script_parameters]
- #id = "#{job[:uuid]}_#{prefix}"
- gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
- gr += edge(job_uuid(job), sp, {:label => prefix}, opts)
end
end
+ gr
end
- gr
- end
- def self.generate_provenance_edges(pdata, visited, uuid, opts)
- gr = ""
- m = ProvenanceHelper::collection_uuid(uuid)
- uuid = m if m
+ def generate_provenance_edges(uuid)
+ gr = ""
+ m = GenerateGraph::collection_uuid(uuid)
+ uuid = m if m
- uuid = uuid.intern if uuid
+ uuid = uuid.intern if uuid
- if (not uuid) or uuid.empty? or visited[uuid]
+ if (not uuid) or uuid.empty? or @visited[uuid]
- #puts "already visited #{uuid}"
- return ""
- end
+ #puts "already @visited #{uuid}"
+ return ""
+ end
- if not pdata[uuid] then
- return ProvenanceHelper::describe_node(pdata, uuid)
- else
- visited[uuid] = true
- end
+ if not @pdata[uuid] then
+ return describe_node(uuid)
+ else
+ @visited[uuid] = true
+ end
- #puts "visiting #{uuid}"
+ #puts "visiting #{uuid}"
- if m
- # uuid is a collection
- gr += ProvenanceHelper::describe_node(pdata, uuid)
+ if m
+ # uuid is a collection
+ gr += describe_node(uuid)
- pdata.each do |k, job|
- if job[:output] == uuid.to_s
- gr += self.edge(uuid, job_uuid(job), {:label => "output"}, opts)
- gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, job[:uuid], opts)
+ @pdata.each do |k, job|
+ if job[:output] == uuid.to_s
+ gr += edge(uuid, job_uuid(job), {:label => "output"})
+ gr += generate_provenance_edges(job[:uuid])
+ end
+ if job[:log] == uuid.to_s
+ gr += edge(uuid, job_uuid(job), {:label => "log"})
+ gr += generate_provenance_edges(job[:uuid])
+ end
end
- if job[:log] == uuid.to_s
- gr += edge(uuid, job_uuid(job), {:label => "log"}, opts)
- gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, job[:uuid], opts)
+ else
+ # uuid is something else
+ rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
+
+ if rsc == Job
+ job = @pdata[uuid]
+ if job
+ gr += script_param_edges(job, "", job[:script_parameters])
+
+ if @opts[:script_version_nodes]
+ gr += edge(job_uuid(job), job[:script_version], {:label => "script_version"})
+ end
+ end
+ else
+ gr += describe_node(uuid)
end
end
- else
- # uuid is something else
- rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
- if rsc == Job
- job = pdata[uuid]
- if job
- gr += ProvenanceHelper::script_param_edges(pdata, visited, job, "", job[:script_parameters], opts)
+ @pdata.each do |k, link|
+ if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
+ gr += describe_node(link[:tail_uuid])
+ gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => "/links/#{link[:uuid]}"})
+ gr += generate_provenance_edges(link[:tail_uuid])
end
- else
- gr += ProvenanceHelper::describe_node(pdata, uuid)
end
+
+ #puts "finished #{uuid}"
+
+ gr
end
- pdata.each do |k, link|
- if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
- gr += ProvenanceHelper::describe_node(pdata, link[:tail_uuid])
- gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => "/links/#{link[:uuid]}"}, opts)
- gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, link[:tail_uuid], opts)
+ def add_jobs_href
+ gr = ""
+ @jobs.each do |k, v|
+ gr += "\"#{k}\" [href=\"/jobs?"
+ script = ""
+ v.each do |u|
+ gr += "uuid%5b%5d=#{u[:uuid]}&"
+ script = u[:script]
+ end
+ gr += "\",label=\""
+ gr += if @opts[:combine_jobs] then "#{script}" else "#{script}\\n#{v[0][:finished_at]}" end
+ gr += "\"];\n"
end
+ gr
end
- #puts "finished #{uuid}"
-
- gr
end
- def self.create_provenance_graph(pdata, uuid, opts={})
- require 'open3'
+ def self.create_provenance_graph(pdata, opts={})
+ if pdata.is_a? Array or pdata.is_a? ArvadosResourceList
+ p2 = {}
+ pdata.each do |k|
+ p2[k[:uuid].intern] = k if k[:uuid]
+ end
+ pdata = p2
+ end
+
+ unless pdata.is_a? Hash
+ raise "create_provenance_graph accepts Array or Hash for pdata only, pdata is #{pdata.class}"
+ end
gr = """strict digraph {
node [fontsize=8,shape=box];
-edge [fontsize=8];"""
+edge [fontsize=8];
+"""
if opts[:direction] == :bottom_up
gr += "edge [dir=back];"
end
- #puts "pdata is #{pdata}"
+ #puts "@pdata is #{pdata}"
- visited = {}
- if uuid.respond_to? :each
- uuid.each do |u|
- gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, u, opts)
- end
- else
- gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, uuid, opts)
+ g = GenerateGraph.new(pdata, opts)
+
+ pdata.each do |k, v|
+ gr += g.generate_provenance_edges(k)
end
+ gr += g.add_jobs_href
+
gr += "}"
svg = ""
#puts gr
+ require 'open3'
+
Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
stdin.print(gr)
stdin.close
@@ -216,4 +274,26 @@ edge [fontsize=8];"""
svg = svg.sub(/<\?xml.*?\?>/m, "")
svg = svg.sub(/<!DOCTYPE.*?>/m, "")
end
+
+ def self.find_collections(sp)
+ c = []
+ if sp and not sp.empty?
+ case sp
+ when Hash
+ sp.each do |k, v|
+ c.concat(find_collections(v))
+ end
+ when Array
+ sp.each do |v|
+ c.concat(find_collections(v))
+ end
+ else
+ m = GenerateGraph::collection_uuid(sp)
+ if m
+ c << m
+ end
+ end
+ end
+ c
+ end
end
diff --git a/apps/workbench/app/views/jobs/index.html.erb b/apps/workbench/app/views/jobs/index.html.erb
index 2188a3b..2c473eb 100644
--- a/apps/workbench/app/views/jobs/index.html.erb
+++ b/apps/workbench/app/views/jobs/index.html.erb
@@ -135,3 +135,5 @@
</tbody>
</table>
+
+<%= raw(@svg) %>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list