[ARVADOS] updated: 1.3.0-1274-g77e32f0b0
Git user
git at public.curoverse.com
Fri Jul 12 15:42:42 UTC 2019
Summary of changes:
.../app/controllers/collections_controller.rb | 4 +-
.../controllers/container_requests_controller.rb | 31 ++---
apps/workbench/app/controllers/jobs_controller.rb | 1 -
.../controllers/pipeline_instances_controller.rb | 1 -
apps/workbench/app/helpers/provenance_helper.rb | 137 +++++++++++++--------
.../arvados/v1/collections_controller.rb | 67 +++++++---
6 files changed, 151 insertions(+), 90 deletions(-)
via 77e32f0b056cc0047a144ee39f54183e8b365584 (commit)
from 6bf53202ce6337c78b1e99cbd6fd6891c3c611f7 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 77e32f0b056cc0047a144ee39f54183e8b365584
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Jul 12 11:41:02 2019 -0400
15422: provenance and used_by endpoints traverse containers
Fix up workbench rendering of provenance as well.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 514101244..de20b8858 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -176,7 +176,7 @@ class CollectionsController < ApplicationController
if params["tab_pane"] == "Provenance_graph"
@prov_svg = ProvenanceHelper::create_provenance_graph(@object.provenance, "provenance_svg",
{:request => request,
- :direction => :top_down,
+ :direction => "RL",
:combine_jobs => :script_only}) rescue nil
end
@@ -217,7 +217,7 @@ class CollectionsController < ApplicationController
if params["tab_pane"] == "Used_by"
@used_by_svg = ProvenanceHelper::create_provenance_graph(@object.used_by, "used_by_svg",
{:request => request,
- :direction => :top_down,
+ :direction => "LR",
:combine_jobs => :script_only,
:pdata_only => true}) rescue nil
end
diff --git a/apps/workbench/app/controllers/container_requests_controller.rb b/apps/workbench/app/controllers/container_requests_controller.rb
index 385d9dc6d..587819dd4 100644
--- a/apps/workbench/app/controllers/container_requests_controller.rb
+++ b/apps/workbench/app/controllers/container_requests_controller.rb
@@ -11,7 +11,7 @@ class ContainerRequestsController < ApplicationController
def generate_provenance(cr)
return if params['tab_pane'] != "Provenance"
- nodes = {cr[:uuid] => cr}
+ nodes = {}
child_crs = []
col_uuids = []
col_pdhs = []
@@ -29,38 +29,33 @@ class ContainerRequestsController < ApplicationController
end
end
- output_cols = {} # Indexed by UUID
- input_cols = {} # Indexed by PDH
+ if nodes.length == 0
+ nodes[cr[:uuid]] = cr
+ end
+
+ pdh_to_col = {} # Indexed by PDH
output_pdhs = []
# Batch requests to get all related collections
# First fetch output collections by UUID.
Collection.filter([['uuid', 'in', col_uuids.uniq]]).each do |c|
- output_cols[c[:uuid]] = c
output_pdhs << c[:portable_data_hash]
+ pdh_to_col[c[:portable_data_hash]] = c
+ nodes[c[:uuid]] = c
end
- # Then, get only input collections by PDH. There could be more than one collection
- # per PDH: the number of collections is used on the collection node label.
+ # Next, get input collections by PDH.
Collection.filter(
[['portable_data_hash', 'in', col_pdhs - output_pdhs]]).each do |c|
- if input_cols[c[:portable_data_hash]]
- input_cols[c[:portable_data_hash]] << c
- else
- input_cols[c[:portable_data_hash]] = [c]
- end
+ nodes[c[:portable_data_hash]] = c
end
@svg = ProvenanceHelper::create_provenance_graph(
nodes, "provenance_svg",
{
:request => request,
- :direction => :top_down,
- :output_collections => output_cols,
- :input_collections => input_cols,
- :cr_children_of => {
- cr[:uuid] => child_crs.select{|child| child[:uuid]},
- },
- })
+ :pdh_to_uuid => pdh_to_col,
+ }
+ )
end
def show_pane_list
diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index 4f7bfcee5..bac1530d3 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -34,7 +34,6 @@ class JobsController < ApplicationController
@svg = ProvenanceHelper::create_provenance_graph nodes, "provenance_svg", {
:request => request,
- :direction => :top_down,
:all_script_parameters => true,
:script_version_nodes => true}
end
diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb
index c8863653a..6431057dd 100644
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@ -192,7 +192,6 @@ class PipelineInstancesController < ApplicationController
if provenance
@prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
:request => request,
- :direction => :top_down,
:all_script_parameters => true,
:combine_jobs => :script_and_version,
:pips => pips,
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
index c06d83ae8..8a8337fe1 100644
--- a/apps/workbench/app/helpers/provenance_helper.rb
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -151,56 +151,97 @@ module ProvenanceHelper
gr
end
- def cr_edges cr, edge_opts={}
+ # def cr_edges cr, edge_opts={}
+ # gr = ""
+
+ # gr += describe_node(cr[:uuid], {href: {controller: 'container_requests',
+ # id: cr[:uuid]},
+ # label: cr[:name],
+ # shape: 'oval'})
+ # # Connect child CRs
+ # children = @opts[:cr_children_of].andand[cr[:uuid]]
+ # if children
+ # children.each do |child|
+ # gr += edge(child[:uuid], cr[:uuid], {label: 'child'})
+ # end
+ # end
+ # # Output collection node
+ # if cr[:output_uuid] and @opts[:output_collections][cr[:output_uuid]]
+ # c = @opts[:output_collections][cr[:output_uuid]]
+ # gr += describe_node(c[:portable_data_hash],
+ # {
+ # label: c[:name],
+ # col_uuid: c[:uuid],
+ # })
+ # gr += edge(cr[:uuid],
+ # c[:portable_data_hash],
+ # {label: 'output'})
+ # end
+ # # Input collection nodes
+ # output_pdhs = @opts[:output_collections].values.collect{|oc|
+ # oc[:portable_data_hash]}
+ # ProvenanceHelper::cr_input_pdhs(cr).each do |pdh|
+ # if not output_pdhs.include?(pdh)
+ # # Search for collections on the same project first
+ # cols = @opts[:input_collections][pdh].andand.select{|ic|
+ # ic[:owner_uuid] == cr[:owner_uuid]}
+ # if not cols or cols.empty?
+ # # Search for any collection with this PDH
+ # cols = @opts[:input_collections][pdh]
+ # end
+ # if cols
+ # names = cols.collect{|x| x[:name]}.uniq
+ # else
+ # names = ['(collection not found)']
+ # end
+ # input_name = names.first
+ # if names.length > 1
+ # input_name += " + #{names.length - 1} more"
+ # end
+ # gr += describe_node(pdh, {label: input_name})
+ # end
+ # gr += edge(pdh, cr[:uuid], {label: 'input'})
+ # end
+
+ # gr
+ # end
+
+ def cr_edges cont, edge_opts={}
+ uuid = cont[:uuid]
gr = ""
- gr += describe_node(cr[:uuid], {href: {controller: 'container_requests',
- id: cr[:uuid]},
- label: cr[:name],
- shape: 'oval'})
- # Connect child CRs
- children = @opts[:cr_children_of].andand[cr[:uuid]]
- if children
- children.each do |child|
- gr += edge(child[:uuid], cr[:uuid], {label: 'child'})
+ gr += describe_node(cont[:uuid], {href: {controller: 'container_requests',
+ id: cont[:uuid]},
+ shape: 'oval',
+ label: cont[:name]})
+
+ ProvenanceHelper::find_collections cont[:mounts] do |collection_hash, collection_uuid, key|
+ if @opts[:pdh_to_uuid] and @opts[:pdh_to_uuid][collection_hash]
+ collection_uuid = @opts[:pdh_to_uuid][collection_hash].uuid
+ collection_hash = nil
+ end
+ if collection_uuid and @pdata[collection_uuid]
+ gr += describe_node(collection_uuid)
+ gr += edge(collection_uuid, uuid, {:label => key})
+ elsif collection_hash and @pdata[collection_hash]
+ gr += describe_node(collection_hash)
+ gr += edge(collection_hash, uuid, {:label => key})
end
end
- # Output collection node
- if cr[:output_uuid] and @opts[:output_collections][cr[:output_uuid]]
- c = @opts[:output_collections][cr[:output_uuid]]
- gr += describe_node(c[:portable_data_hash],
- {
- label: c[:name],
- col_uuid: c[:uuid],
- })
- gr += edge(cr[:uuid],
- c[:portable_data_hash],
- {label: 'output'})
+
+ if cont[:container_image] and !@opts[:no_docker] and @pdata[cont[:container_image]]
+ gr += describe_node(cont[:container_image], {label: cont[:container_image]})
+ gr += edge(cont[:container_image], uuid, {label: "docker_image"})
end
- # Input collection nodes
- output_pdhs = @opts[:output_collections].values.collect{|oc|
- oc[:portable_data_hash]}
- ProvenanceHelper::cr_input_pdhs(cr).each do |pdh|
- if not output_pdhs.include?(pdh)
- # Search for collections on the same project first
- cols = @opts[:input_collections][pdh].andand.select{|ic|
- ic[:owner_uuid] == cr[:owner_uuid]}
- if not cols or cols.empty?
- # Search for any collection with this PDH
- cols = @opts[:input_collections][pdh]
- end
- if cols
- names = cols.collect{|x| x[:name]}.uniq
- else
- names = ['(collection not found)']
- end
- input_name = names.first
- if names.length > 1
- input_name += " + #{names.length - 1} more"
- end
- gr += describe_node(pdh, {label: input_name})
- end
- gr += edge(pdh, cr[:uuid], {label: 'input'})
+
+ if cont[:output_uuid] and !edge_opts[:no_output] and @pdata[cont[:output_uuid]]
+ gr += describe_node(cont[:output_uuid])
+ gr += edge(uuid, cont[:output_uuid], {label: "output" })
+ end
+
+ if cont[:log_uuid] and !edge_opts[:no_log] and @pdata[cont[:log_uuid]]
+ gr += describe_node(cont[:log_uuid])
+ gr += edge(uuid, cont[:log_uuid], {label: "log"})
end
gr
@@ -364,11 +405,9 @@ module ProvenanceHelper
gr = """strict digraph {
node [fontsize=10,fontname=\"Helvetica,Arial,sans-serif\"];
edge [fontsize=10,fontname=\"Helvetica,Arial,sans-serif\"];
-rankdir=RL;
"""
-
- if opts[:direction] == :bottom_up
- gr += "edge [dir=back];"
+ if ["LR", "RL"].include? opts[:direction]
+ gr += "rankdir=#{opts[:direction]};"
end
begin
diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 51a47f018..ede6fbd89 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -96,11 +96,11 @@ class Arvados::V1::CollectionsController < ApplicationController
end
- def find_collections(visited, sp, &b)
+ def find_collections(visited, sp, ignore_columns=[], &b)
case sp
when ArvadosModel
sp.class.columns.each do |c|
- find_collections(visited, sp[c.name.to_sym], &b) if c.name != "log"
+ find_collections(visited, sp[c.name.to_sym], &b) if !ignore_columns.include?(c.name)
end
when Hash
sp.each do |k, v|
@@ -129,8 +129,6 @@ class Arvados::V1::CollectionsController < ApplicationController
return if visited[loc.to_s]
end
- logger.debug "visiting #{uuid}"
-
if loc
# uuid is a portable_data_hash
collections = Collection.readable_by(*@read_users).where(portable_data_hash: loc.to_s)
@@ -188,14 +186,11 @@ class Arvados::V1::CollectionsController < ApplicationController
end
end
- Container.readable_by(*@read_users).where(["mounts like ?", "%#{loc.to_s}%"]).each do |c|
- search_edges(visited, c.uuid, :search_down)
- end
-
- Container.readable_by(*@read_users).where(["container_image = '#{loc.to_s}'"]).each do |c|
- search_edges(visited, c.uuid, :search_down)
+ Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).each do |c|
+ if c.output != loc.to_s && c.log != loc.to_s
+ search_edges(visited, c.uuid, :search_down)
+ end
end
-
end
else
# uuid is a regular Arvados UUID
@@ -215,7 +210,8 @@ class Arvados::V1::CollectionsController < ApplicationController
end
end
elsif rsc == Container
- Container.readable_by(*@read_users).where(uuid: uuid).each do |c|
+ c = Container.readable_by(*@read_users).where(uuid: uuid).limit(1).first
+ if c
visited[uuid] = c.as_api_response
if direction == :search_up
# Follow upstream collections referenced in the script parameters
@@ -228,10 +224,37 @@ class Arvados::V1::CollectionsController < ApplicationController
search_edges(visited, c.output, direction)
end
end
+ elsif rsc == ContainerRequest
+ c = ContainerRequest.readable_by(*@read_users).where(uuid: uuid).limit(1).first
+ if c
+ visited[uuid] = c.as_api_response
+ if direction == :search_up
+ # Follow upstream collections
+ find_collections(visited, c, ignore_columns=["log_uuid"]) do |hash, col_uuid|
+ search_edges(visited, hash, :search_up) if hash
+ search_edges(visited, col_uuid, :search_up) if col_uuid
+ end
+ elsif direction == :search_down
+ # Follow downstream job output
+ search_edges(visited, c.output_uuid, direction)
+ end
+ end
elsif rsc == Collection
- if c = Collection.readable_by(*@read_users).where(uuid: uuid).limit(1).first
- search_edges(visited, c.portable_data_hash, direction)
- visited[c.portable_data_hash] = c.as_api_response
+ c = Collection.readable_by(*@read_users).where(uuid: uuid).limit(1).first
+ if c
+ if direction == :search_up
+ visited[c.uuid] = c.as_api_response
+
+ ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).each do |cr|
+ search_edges(visited, cr.uuid, :search_up)
+ end
+
+ ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).each do |cr|
+ search_edges(visited, cr.uuid, :search_up)
+ end
+ elsif direction == :search_down
+ search_edges(visited, c.portable_data_hash, direction)
+ end
end
elsif rsc != nil
rsc.where(uuid: uuid).each do |r|
@@ -261,15 +284,21 @@ class Arvados::V1::CollectionsController < ApplicationController
def provenance
visited = {}
- search_edges(visited, @object[:portable_data_hash], :search_up)
- search_edges(visited, @object[:uuid], :search_up)
+ if @object[:uuid]
+ search_edges(visited, @object[:uuid], :search_up)
+ else
+ search_edges(visited, @object[:portable_data_hash], :search_up)
+ end
send_json visited
end
def used_by
visited = {}
- search_edges(visited, @object[:uuid], :search_down)
- search_edges(visited, @object[:portable_data_hash], :search_down)
+ if @object[:uuid]
+ search_edges(visited, @object[:uuid], :search_down)
+ else
+ search_edges(visited, @object[:portable_data_hash], :search_down)
+ end
send_json visited
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list