[ARVADOS] updated: 4c8468e142d92a4dd063bde9ea1337b2161999e9
git at public.curoverse.com
git at public.curoverse.com
Wed Jan 29 16:57:17 EST 2014
Summary of changes:
.../app/controllers/collections_controller.rb | 142 +-------------
.../controllers/pipeline_instances_controller.rb | 25 +++
apps/workbench/app/helpers/provenance_helper.rb | 219 ++++++++++++++++++++
apps/workbench/app/views/collections/show.html.erb | 4 +-
.../app/views/pipeline_instances/show.html.erb | 3 +
docker/run.sh | 18 +-
sdk/cli/Gemfile.lock | 14 +-
sdk/cli/arvados-cli.gemspec | 14 +-
sdk/python/arvados/__init__.py | 2 +-
.../arvados/v1/collections_controller.rb | 11 +-
services/api/app/models/job.rb | 16 +-
.../20140117231056_normalize_collection_uuid.rb | 4 +-
...malize_collection_uuids_in_script_parameters.rb | 45 ++++
services/api/db/schema.rb | 2 +-
14 files changed, 339 insertions(+), 180 deletions(-)
create mode 100644 apps/workbench/app/helpers/provenance_helper.rb
create mode 100644 services/api/db/migrate/20140129184311_normalize_collection_uuids_in_script_parameters.rb
via 4c8468e142d92a4dd063bde9ea1337b2161999e9 (commit)
via 604d8ddb533176dc2d067189e11376dc8dc81af3 (commit)
via 3acff4c57e3459dae420903c02c717cd5c07319f (commit)
via e3e224295f2a3aa53ee82ebfc3982b90e85cfc08 (commit)
via c67624630594b481aa96d548282187720601abea (commit)
via e2b7fd4df7f6ba211f2bdbc173ff9cf5de61fbb0 (commit)
via 53f098a4343081d2f31be9b84c11973320532337 (commit)
via 31d298b5a3550b02d0f67c6e6110821cd078fbb2 (commit)
via 3409236dee75142911976ebaef134ae203a726c6 (commit)
via 2728f59746f7a57ed0283b899f9636cfef77d001 (commit)
via 566bb36cbadbe0e4ceb1a30123033d9310a0f4d8 (commit)
via 5f7fa947b13eeee04d56b487f7173308635aa60c (commit)
via ca5197de7e459a48d72cda8e1f4de4c4201366fb (commit)
via c106ef5033c4fcf19e6409fa517030ecffb3e2ea (commit)
via 68a05d89a28f609e3592d241bf6d0417c75d219a (commit)
from 04bb94072486e32ed9784be3a22c43d961c84bee (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4c8468e142d92a4dd063bde9ea1337b2161999e9
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jan 29 16:58:03 2014 -0500
Now renders collections using up to the first three files in the collection
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
index de3401d..dbc7dbb 100644
--- a/apps/workbench/app/helpers/provenance_helper.rb
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -11,17 +11,26 @@ module ProvenanceHelper
if pdata[uuid][:name]
return "\"#{uuid}\" [label=\"#{pdata[uuid][:name]}\",href=\"#{href}\",shape=oval];\n"
else
- i = 0
- label = ""
- while i < 3 and i < pdata[uuid].files.length
- label += "\\n" unless label == ""
- label += pdata[uuid].files[i][1]
- i += 1
+ files = nil
+ if pdata[uuid].respond_to? :files
+ files = pdata[uuid].files
+ elsif pdata[uuid][:files]
+ files = pdata[uuid][:files]
end
- if i < pdata[uuid].files.length
- label += "\\n⋮"
+
+ if files
+ i = 0
+ label = ""
+ while i < 3 and i < files.length
+ label += "\\n" unless label == ""
+ label += files[i][1]
+ i += 1
+ end
+ if i < files.length
+ label += "\\n⋮"
+ end
+ return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval];\n"
end
- return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval];\n"
end
end
return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\"];\n"
@@ -88,9 +97,9 @@ module ProvenanceHelper
unless node == ""
node += "']"
#puts node
- id = "#{job[:uuid]}_#{prefix}"
- gr += "\"#{id}\" [label=\"#{node}\"];\n"
- gr += edge(job_uuid(job), id, {:label => prefix}, opts)
+ #id = "#{job[:uuid]}_#{prefix}"
+ gr += "\"#{node}\" [label=\"#{node}\"];\n"
+ gr += edge(job_uuid(job), node, {:label => prefix}, opts)
end
else
m = collection_uuid(sp)
@@ -98,9 +107,9 @@ module ProvenanceHelper
gr += edge(job_uuid(job), m, {:label => prefix}, opts)
gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, m, opts)
elsif opts[:all_script_parameters]
- id = "#{job[:uuid]}_#{prefix}"
- gr += "\"#{id}\" [label=\"#{sp}\"];\n"
- gr += edge(job_uuid(job), id, {:label => prefix}, opts)
+ #id = "#{job[:uuid]}_#{prefix}"
+ gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
+ gr += edge(job_uuid(job), sp, {:label => prefix}, opts)
end
end
end
diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index ad2d6d1..9198f58 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -93,6 +93,10 @@ class Arvados::V1::CollectionsController < ApplicationController
# uuid is a collection
Collection.where(uuid: uuid).each do |c|
visited[uuid] = c.as_api_response
+ visited[uuid][:files] = []
+ c.files.each do |f|
+ visited[uuid][:files] << f
+ end
end
Job.where(output: uuid).each do |job|
commit 604d8ddb533176dc2d067189e11376dc8dc81af3
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jan 29 16:36:53 2014 -0500
Very nice provenance diagram for pipelines.
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index ca176b4..27f5bb7 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -99,7 +99,7 @@ class CollectionsController < ApplicationController
end
Collection.where(uuid: @object.uuid).each do |u|
- @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, u.uuid
+ @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, u.uuid, {:direction => :bottom_up}
end
end
diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb
index 79aade7..37e743f 100644
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@ -8,19 +8,19 @@ class PipelineInstancesController < ApplicationController
j = v[:job]
somejob = j[:uuid]
provenance[somejob.intern] = j
- collections << j[:output]
+ collections << j[:output].intern
+ j[:dependencies].each do |k|
+ collections << k.intern
+ end
end
- puts collections
- puts '---'
-
Collection.where(uuid: collections).each do |c|
- puts c.uuid
+ #puts c.uuid
provenance[c.uuid.intern] = c
end
PipelineInstance.where(uuid: @object.uuid).each do |u|
- @prov_svg = CollectionsController::create_provenance_graph provenance, somejob
+ @prov_svg = ProvenanceHelper::create_provenance_graph provenance, collections, {:all_script_parameters => true}
end
end
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
index c393a1b..de3401d 100644
--- a/apps/workbench/app/helpers/provenance_helper.rb
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -1,12 +1,33 @@
-class ProvenanceHelper
- def self.describe_node(uuid)
- uuid = uuid.to_s
- rsc = ArvadosBase::resource_class_for_uuid uuid
+module ProvenanceHelper
+ def self.describe_node(pdata, uuid)
+ rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
if rsc
- "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
- else
- ""
+ href = "/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}"
+
+ #"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
+ if rsc == Collection
+ if pdata[uuid]
+ #puts pdata[uuid]
+ if pdata[uuid][:name]
+ return "\"#{uuid}\" [label=\"#{pdata[uuid][:name]}\",href=\"#{href}\",shape=oval];\n"
+ else
+ i = 0
+ label = ""
+ while i < 3 and i < pdata[uuid].files.length
+ label += "\\n" unless label == ""
+ label += pdata[uuid].files[i][1]
+ i += 1
+ end
+ if i < pdata[uuid].files.length
+ label += "\\n⋮"
+ end
+ return "\"#{uuid}\" [label=\"#{label}\",href=\"#{href}\",shape=oval];\n"
+ end
+ end
+ return "\"#{uuid}\" [label=\"#{rsc}\",href=\"#{href}\"];\n"
+ end
end
+ return ""
end
def self.job_uuid(job)
@@ -23,7 +44,24 @@ class ProvenanceHelper
end
end
- def self.script_param_edges(visited, job, prefix, sp, opts)
+ def self.edge(tail, head, extra, opts)
+ if opts[:direction] == :bottom_up
+ gr = "\"#{tail}\" -> \"#{head}\""
+ else
+ gr = "\"#{head}\" -> \"#{tail}\""
+ end
+ if extra.length > 0
+ gr += "["
+ extra.each do |k, v|
+ gr += "#{k}=\"#{v}\","
+ end
+ gr += "]"
+ end
+ gr += ";\n"
+ gr
+ end
+
+ def self.script_param_edges(pdata, visited, job, prefix, sp, opts)
gr = ""
if sp and not sp.empty?
case sp
@@ -32,57 +70,76 @@ class ProvenanceHelper
if prefix.size > 0
k = prefix + "::" + k.to_s
end
- gr += CollectionsController::script_param_edges(visited, job, k.to_s, v, opts)
+ gr += ProvenanceHelper::script_param_edges(pdata, visited, job, k.to_s, v, opts)
end
when Array
+ i = 0
+ node = ""
sp.each do |v|
- gr += CollectionsController::script_param_edges(visited, job, prefix, v, opts)
+ if collection_uuid(v)
+ gr += ProvenanceHelper::script_param_edges(pdata, visited, job, "#{prefix}[#{i}]", v, opts)
+ else
+ node += "', '" unless node == ""
+ node = "['" if node == ""
+ node += "#{v}"
+ end
+ i += 1
+ end
+ unless node == ""
+ node += "']"
+ #puts node
+ id = "#{job[:uuid]}_#{prefix}"
+ gr += "\"#{id}\" [label=\"#{node}\"];\n"
+ gr += edge(job_uuid(job), id, {:label => prefix}, opts)
end
else
m = collection_uuid(sp)
if m
- gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
- gr += CollectionsController::generate_provenance_edges(visited, m, opts)
+ gr += edge(job_uuid(job), m, {:label => prefix}, opts)
+ gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, m, opts)
+ elsif opts[:all_script_parameters]
+ id = "#{job[:uuid]}_#{prefix}"
+ gr += "\"#{id}\" [label=\"#{sp}\"];\n"
+ gr += edge(job_uuid(job), id, {:label => prefix}, opts)
end
end
end
gr
end
- def self.generate_provenance_edges(pdata, uuid, opts)
+ def self.generate_provenance_edges(pdata, visited, uuid, opts)
gr = ""
- m = CollectionsController::collection_uuid(uuid)
+ m = ProvenanceHelper::collection_uuid(uuid)
uuid = m if m
uuid = uuid.intern if uuid
- if (not uuid) or uuid.empty? \
- or (pdata[uuid] and pdata[uuid][:_visited])
+ if (not uuid) or uuid.empty? or visited[uuid]
#puts "already visited #{uuid}"
return ""
end
if not pdata[uuid] then
- return CollectionsController::describe_node(uuid)
+ return ProvenanceHelper::describe_node(pdata, uuid)
else
- pdata[uuid][:_visited] = true
+ visited[uuid] = true
end
#puts "visiting #{uuid}"
if m
# uuid is a collection
- gr += CollectionsController::describe_node(uuid)
+ gr += ProvenanceHelper::describe_node(pdata, uuid)
pdata.each do |k, job|
if job[:output] == uuid.to_s
- gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"output\"];"
- gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
+ gr += self.edge(uuid, job_uuid(job), {:label => "output"}, opts)
+ gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, job[:uuid], opts)
end
if job[:log] == uuid.to_s
- gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"log\"];"
- gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
+ gr += edge(uuid, job_uuid(job), {:label => "log"}, opts)
+ gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, job[:uuid], opts)
end
end
else
@@ -92,18 +149,18 @@ class ProvenanceHelper
if rsc == Job
job = pdata[uuid]
if job
- gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters], opts)
+ gr += ProvenanceHelper::script_param_edges(pdata, visited, job, "", job[:script_parameters], opts)
end
else
- gr += CollectionsController::describe_node(uuid)
+ gr += ProvenanceHelper::describe_node(pdata, uuid)
end
end
pdata.each do |k, link|
if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
- gr += CollectionsController::describe_node(link[:tail_uuid])
- gr += "\"#{link[:head_uuid]}\" -> \"#{link[:tail_uuid]}\" [label=\" #{link[:name]}\", href=\"/links/#{link[:uuid]}\"];"
- gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid], opts)
+ gr += ProvenanceHelper::describe_node(pdata, link[:tail_uuid])
+ gr += edge(link[:head_uuid], link[:tail_uuid], {:label => link[:name], :href => "/links/#{link[:uuid]}"}, opts)
+ gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, link[:tail_uuid], opts)
end
end
@@ -117,15 +174,28 @@ class ProvenanceHelper
gr = """strict digraph {
node [fontsize=8,shape=box];
-edge [dir=back,fontsize=8];"""
+edge [fontsize=8];"""
+
+ if opts[:direction] == :bottom_up
+ gr += "edge [dir=back];"
+ end
#puts "pdata is #{pdata}"
- gr += CollectionsController::generate_provenance_edges(pdata, uuid, opts)
+ visited = {}
+ if uuid.respond_to? :each
+ uuid.each do |u|
+ gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, u, opts)
+ end
+ else
+ gr += ProvenanceHelper::generate_provenance_edges(pdata, visited, uuid, opts)
+ end
gr += "}"
svg = ""
+ #puts gr
+
Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
stdin.print(gr)
stdin.close
commit 3acff4c57e3459dae420903c02c717cd5c07319f
Merge: e3e2242 c676246
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jan 29 14:58:53 2014 -0500
Merge remote-tracking branch 'origin/master' into 1977-provenance-report
commit e3e224295f2a3aa53ee82ebfc3982b90e85cfc08
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jan 29 14:58:11 2014 -0500
Started refactoring
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 3a1013a..ca176b4 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -52,144 +52,6 @@ class CollectionsController < ApplicationController
self.response_body = FileStreamer.new opts
end
- def self.describe_node(uuid)
- uuid = uuid.to_s
- rsc = ArvadosBase::resource_class_for_uuid uuid
- if rsc
- "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
- else
- ""
- end
- end
-
- def self.job_uuid(job)
- # "#{job[:script]}\\n#{job[:script_version]}"
- "#{job[:script]}"
- end
-
- def self.collection_uuid(uuid)
- m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid.to_s)
- if m
- m[1]
- else
- nil
- end
- end
-
- def self.script_param_edges(visited, job, prefix, sp, opts)
- gr = ""
- if sp and not sp.empty?
- case sp
- when Hash
- sp.each do |k, v|
- if prefix.size > 0
- k = prefix + "::" + k.to_s
- end
- gr += CollectionsController::script_param_edges(visited, job, k.to_s, v, opts)
- end
- when Array
- sp.each do |v|
- gr += CollectionsController::script_param_edges(visited, job, prefix, v, opts)
- end
- else
- m = collection_uuid(sp)
- if m
- gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
- gr += CollectionsController::generate_provenance_edges(visited, m, opts)
- end
- end
- end
- gr
- end
-
- def self.generate_provenance_edges(pdata, uuid, opts)
- gr = ""
- m = CollectionsController::collection_uuid(uuid)
- uuid = m if m
-
- uuid = uuid.intern if uuid
-
- if (not uuid) or uuid.empty? \
- or (pdata[uuid] and pdata[uuid][:_visited])
-
- #puts "already visited #{uuid}"
- return ""
- end
-
- if not pdata[uuid] then
- return CollectionsController::describe_node(uuid)
- else
- pdata[uuid][:_visited] = true
- end
-
- #puts "visiting #{uuid}"
-
- if m
- # uuid is a collection
- gr += CollectionsController::describe_node(uuid)
-
- pdata.each do |k, job|
- if job[:output] == uuid.to_s
- gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"output\"];"
- gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
- end
- if job[:log] == uuid.to_s
- gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"log\"];"
- gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
- end
- end
- else
- # uuid is something else
- rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
-
- if rsc == Job
- job = pdata[uuid]
- if job
- gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters], opts)
- end
- else
- gr += CollectionsController::describe_node(uuid)
- end
- end
-
- pdata.each do |k, link|
- if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
- gr += CollectionsController::describe_node(link[:tail_uuid])
- gr += "\"#{link[:head_uuid]}\" -> \"#{link[:tail_uuid]}\" [label=\" #{link[:name]}\", href=\"/links/#{link[:uuid]}\"];"
- gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid], opts)
- end
- end
-
- #puts "finished #{uuid}"
-
- gr
- end
-
- def self.create_provenance_graph(pdata, uuid, opts={})
- require 'open3'
-
- gr = """strict digraph {
-node [fontsize=8,shape=box];
-edge [dir=back,fontsize=8];"""
-
- #puts "pdata is #{pdata}"
-
- gr += CollectionsController::generate_provenance_edges(pdata, uuid, opts)
-
- gr += "}"
- svg = ""
-
- Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
- stdin.print(gr)
- stdin.close
- svg = stdout.read()
- wait_thr.value
- stdout.close()
- end
-
- svg = svg.sub(/<\?xml.*?\?>/m, "")
- svg = svg.sub(/<!DOCTYPE.*?>/m, "")
- end
def show
return super if !@object
@@ -237,7 +99,7 @@ edge [dir=back,fontsize=8];"""
end
Collection.where(uuid: @object.uuid).each do |u|
- @prov_svg = CollectionsController::create_provenance_graph u.provenance, u.uuid
+ @prov_svg = ProvenanceHelper::create_provenance_graph u.provenance, u.uuid
end
end
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
new file mode 100644
index 0000000..c393a1b
--- /dev/null
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -0,0 +1,140 @@
+class ProvenanceHelper
+ def self.describe_node(uuid)
+ uuid = uuid.to_s
+ rsc = ArvadosBase::resource_class_for_uuid uuid
+ if rsc
+ "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
+ else
+ ""
+ end
+ end
+
+ def self.job_uuid(job)
+ # "#{job[:script]}\\n#{job[:script_version]}"
+ "#{job[:script]}"
+ end
+
+ def self.collection_uuid(uuid)
+ m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid.to_s)
+ if m
+ m[1]
+ else
+ nil
+ end
+ end
+
+ def self.script_param_edges(visited, job, prefix, sp, opts)
+ gr = ""
+ if sp and not sp.empty?
+ case sp
+ when Hash
+ sp.each do |k, v|
+ if prefix.size > 0
+ k = prefix + "::" + k.to_s
+ end
+ gr += CollectionsController::script_param_edges(visited, job, k.to_s, v, opts)
+ end
+ when Array
+ sp.each do |v|
+ gr += CollectionsController::script_param_edges(visited, job, prefix, v, opts)
+ end
+ else
+ m = collection_uuid(sp)
+ if m
+ gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
+ gr += CollectionsController::generate_provenance_edges(visited, m, opts)
+ end
+ end
+ end
+ gr
+ end
+
+ def self.generate_provenance_edges(pdata, uuid, opts)
+ gr = ""
+ m = CollectionsController::collection_uuid(uuid)
+ uuid = m if m
+
+ uuid = uuid.intern if uuid
+
+ if (not uuid) or uuid.empty? \
+ or (pdata[uuid] and pdata[uuid][:_visited])
+
+ #puts "already visited #{uuid}"
+ return ""
+ end
+
+ if not pdata[uuid] then
+ return CollectionsController::describe_node(uuid)
+ else
+ pdata[uuid][:_visited] = true
+ end
+
+ #puts "visiting #{uuid}"
+
+ if m
+ # uuid is a collection
+ gr += CollectionsController::describe_node(uuid)
+
+ pdata.each do |k, job|
+ if job[:output] == uuid.to_s
+ gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"output\"];"
+ gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
+ end
+ if job[:log] == uuid.to_s
+ gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"log\"];"
+ gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
+ end
+ end
+ else
+ # uuid is something else
+ rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
+
+ if rsc == Job
+ job = pdata[uuid]
+ if job
+ gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters], opts)
+ end
+ else
+ gr += CollectionsController::describe_node(uuid)
+ end
+ end
+
+ pdata.each do |k, link|
+ if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
+ gr += CollectionsController::describe_node(link[:tail_uuid])
+ gr += "\"#{link[:head_uuid]}\" -> \"#{link[:tail_uuid]}\" [label=\" #{link[:name]}\", href=\"/links/#{link[:uuid]}\"];"
+ gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid], opts)
+ end
+ end
+
+ #puts "finished #{uuid}"
+
+ gr
+ end
+
+ def self.create_provenance_graph(pdata, uuid, opts={})
+ require 'open3'
+
+ gr = """strict digraph {
+node [fontsize=8,shape=box];
+edge [dir=back,fontsize=8];"""
+
+ #puts "pdata is #{pdata}"
+
+ gr += CollectionsController::generate_provenance_edges(pdata, uuid, opts)
+
+ gr += "}"
+ svg = ""
+
+ Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
+ stdin.print(gr)
+ stdin.close
+ svg = stdout.read()
+ wait_thr.value
+ stdout.close()
+ end
+
+ svg = svg.sub(/<\?xml.*?\?>/m, "")
+ svg = svg.sub(/<!DOCTYPE.*?>/m, "")
+ end
+end
commit e2b7fd4df7f6ba211f2bdbc173ff9cf5de61fbb0
Merge: 31d298b 53f098a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jan 29 14:41:55 2014 -0500
Merge branch 'master' into 1977-provenance-report
commit 31d298b5a3550b02d0f67c6e6110821cd078fbb2
Merge: 3409236 2728f59
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jan 29 14:15:25 2014 -0500
Merge commit '2728f59' into 1977-provenance-report
commit 3409236dee75142911976ebaef134ae203a726c6
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jan 29 14:14:36 2014 -0500
Working on visualization of pipelines
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index dcf1d91..3a1013a 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -58,7 +58,6 @@ class CollectionsController < ApplicationController
if rsc
"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
else
- puts "error! #{uuid} #{rsc}"
""
end
end
@@ -77,7 +76,7 @@ class CollectionsController < ApplicationController
end
end
- def self.script_param_edges(visited, job, prefix, sp)
+ def self.script_param_edges(visited, job, prefix, sp, opts)
gr = ""
if sp and not sp.empty?
case sp
@@ -86,24 +85,24 @@ class CollectionsController < ApplicationController
if prefix.size > 0
k = prefix + "::" + k.to_s
end
- gr += CollectionsController::script_param_edges(visited, job, k.to_s, v)
+ gr += CollectionsController::script_param_edges(visited, job, k.to_s, v, opts)
end
when Array
sp.each do |v|
- gr += CollectionsController::script_param_edges(visited, job, prefix, v)
+ gr += CollectionsController::script_param_edges(visited, job, prefix, v, opts)
end
else
m = collection_uuid(sp)
if m
gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
- gr += CollectionsController::generate_provenance_edges(visited, m)
+ gr += CollectionsController::generate_provenance_edges(visited, m, opts)
end
end
end
gr
end
- def self.generate_provenance_edges(pdata, uuid)
+ def self.generate_provenance_edges(pdata, uuid, opts)
gr = ""
m = CollectionsController::collection_uuid(uuid)
uuid = m if m
@@ -113,7 +112,7 @@ class CollectionsController < ApplicationController
if (not uuid) or uuid.empty? \
or (pdata[uuid] and pdata[uuid][:_visited])
- puts "already visited #{uuid}"
+ #puts "already visited #{uuid}"
return ""
end
@@ -123,7 +122,7 @@ class CollectionsController < ApplicationController
pdata[uuid][:_visited] = true
end
- puts "visiting #{uuid}"
+ #puts "visiting #{uuid}"
if m
# uuid is a collection
@@ -146,7 +145,7 @@ class CollectionsController < ApplicationController
if rsc == Job
job = pdata[uuid]
if job
- gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters])
+ gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters], opts)
end
else
gr += CollectionsController::describe_node(uuid)
@@ -157,25 +156,25 @@ class CollectionsController < ApplicationController
if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
gr += CollectionsController::describe_node(link[:tail_uuid])
gr += "\"#{link[:head_uuid]}\" -> \"#{link[:tail_uuid]}\" [label=\" #{link[:name]}\", href=\"/links/#{link[:uuid]}\"];"
- gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid])
+ gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid], opts)
end
end
- puts "finished #{uuid}"
+ #puts "finished #{uuid}"
gr
end
- def self.create_provenance_graph(pdata, uuid)
+ def self.create_provenance_graph(pdata, uuid, opts={})
require 'open3'
gr = """strict digraph {
node [fontsize=8,shape=box];
edge [dir=back,fontsize=8];"""
- puts "pdata is #{pdata}"
+ #puts "pdata is #{pdata}"
- gr += CollectionsController::generate_provenance_edges(pdata, uuid)
+ gr += CollectionsController::generate_provenance_edges(pdata, uuid, opts)
gr += "}"
svg = ""
diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb
index afb9949..79aade7 100644
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@ -1,2 +1,27 @@
class PipelineInstancesController < ApplicationController
+
+ def show
+ provenance = {}
+ somejob = nil
+ collections = []
+ @object.components.each do |k, v|
+ j = v[:job]
+ somejob = j[:uuid]
+ provenance[somejob.intern] = j
+ collections << j[:output]
+ end
+
+ puts collections
+ puts '---'
+
+ Collection.where(uuid: collections).each do |c|
+ puts c.uuid
+ provenance[c.uuid.intern] = c
+ end
+
+ PipelineInstance.where(uuid: @object.uuid).each do |u|
+ @prov_svg = CollectionsController::create_provenance_graph provenance, somejob
+ end
+ end
+
end
diff --git a/apps/workbench/app/views/pipeline_instances/show.html.erb b/apps/workbench/app/views/pipeline_instances/show.html.erb
index 74c090d..2745e76 100644
--- a/apps/workbench/app/views/pipeline_instances/show.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/show.html.erb
@@ -28,6 +28,9 @@
<% end %>
</tbody>
</table>
+
+<%= raw(@prov_svg) %>
+
<pre>
<%= JSON.pretty_generate(@object.attributes) rescue nil %>
</pre>
commit 5f7fa947b13eeee04d56b487f7173308635aa60c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Jan 29 11:38:27 2014 -0500
Fixed bugs in creating graph from API provenance report.
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 57b9bab..dcf1d91 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -58,18 +58,14 @@ class CollectionsController < ApplicationController
if rsc
"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
else
+ puts "error! #{uuid} #{rsc}"
""
end
end
- def self.describe_script(job)
- # """\"#{job.script_version}\" [label=\"#{job.script}: #{job.script_version}\"];
- # \"#{job.uuid}\" -> \"#{job.script_version}\" [label=\"script\"];"""
- "\"#{job.uuid}\" [label=\"#{job.script}\\n#{job.script_version}\"];"
- end
-
def self.job_uuid(job)
- "#{job.script}\\n#{job.script_version}"
+ # "#{job[:script]}\\n#{job[:script_version]}"
+ "#{job[:script]}"
end
def self.collection_uuid(uuid)
@@ -115,39 +111,42 @@ class CollectionsController < ApplicationController
uuid = uuid.intern if uuid
if (not uuid) or uuid.empty? \
- or (pdata[uuid] and pdata[uuid][:_visited]) \
- or (not pdata[uuid])
+ or (pdata[uuid] and pdata[uuid][:_visited])
puts "already visited #{uuid}"
return ""
end
- puts "visiting #{uuid}"
+ if not pdata[uuid] then
+ return CollectionsController::describe_node(uuid)
+ else
+ pdata[uuid][:_visited] = true
+ end
- pdata[uuid][:_visited] = true
+ puts "visiting #{uuid}"
if m
# uuid is a collection
gr += CollectionsController::describe_node(uuid)
pdata.each do |k, job|
- if job[:output] == uuid
+ if job[:output] == uuid.to_s
gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"output\"];"
gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
end
- if job[:log] == uuid
+ if job[:log] == uuid.to_s
gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"log\"];"
gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
end
end
else
# uuid is something else
- rsc = ArvadosBase::resource_class_for_uuid uuid
+ rsc = ArvadosBase::resource_class_for_uuid uuid.to_s
if rsc == Job
job = pdata[uuid]
if job
- gr += CollectionsController::script_param_edges(pdata, job, "", job.script_parameters)
+ gr += CollectionsController::script_param_edges(pdata, job, "", job[:script_parameters])
end
else
gr += CollectionsController::describe_node(uuid)
diff --git a/apps/workbench/app/views/collections/show.html.erb b/apps/workbench/app/views/collections/show.html.erb
index adb8bf0..8c6e83e 100644
--- a/apps/workbench/app/views/collections/show.html.erb
+++ b/apps/workbench/app/views/collections/show.html.erb
@@ -210,8 +210,8 @@
</td><td>
<%= j.script_version %>
</td><td>
- <span class="label <%= if j.success then 'label-success'; elsif j.active then 'label-primary'; else 'label-warning'; end %>">
- <%= j.success || j.active ? 'ok' : 'failed' %>
+ <span class="label <%= if j.success then 'label-success'; elsif j.running then 'label-primary'; else 'label-warning'; end %>">
+ <%= j.success || j.running ? 'ok' : 'failed' %>
</span>
</td><td>
<%= j.started_at %>
diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 48544a2..ad2d6d1 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -77,13 +77,13 @@ class Arvados::V1::CollectionsController < ApplicationController
end
end
end
- gr
end
def generate_provenance_edges(visited, uuid)
m = collection_uuid(uuid)
+ uuid = m if m
- if not uuid or uuid.empty? or visited[uuid] or visited[m]
+ if not uuid or uuid.empty? or visited[uuid]
return ""
end
@@ -91,7 +91,6 @@ class Arvados::V1::CollectionsController < ApplicationController
if m
# uuid is a collection
- uuid = m
Collection.where(uuid: uuid).each do |c|
visited[uuid] = c.as_api_response
end
@@ -110,7 +109,7 @@ class Arvados::V1::CollectionsController < ApplicationController
if rsc == Job
Job.where(uuid: uuid).each do |job|
visited[uuid] = job.as_api_response
- script_param_edges(visited, job, "", job.script_parameters)
+ script_param_edges(visited, job.script_parameters)
end
elsif rsc != nil
rsc.where(uuid: uuid).each do |r|
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list