[ARVADOS] updated: 1.3.0-1274-g77e32f0b0

Git user git at public.curoverse.com
Fri Jul 12 15:42:42 UTC 2019


Summary of changes:
 .../app/controllers/collections_controller.rb      |   4 +-
 .../controllers/container_requests_controller.rb   |  31 ++---
 apps/workbench/app/controllers/jobs_controller.rb  |   1 -
 .../controllers/pipeline_instances_controller.rb   |   1 -
 apps/workbench/app/helpers/provenance_helper.rb    | 137 +++++++++++++--------
 .../arvados/v1/collections_controller.rb           |  67 +++++++---
 6 files changed, 151 insertions(+), 90 deletions(-)

       via  77e32f0b056cc0047a144ee39f54183e8b365584 (commit)
      from  6bf53202ce6337c78b1e99cbd6fd6891c3c611f7 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 77e32f0b056cc0047a144ee39f54183e8b365584
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Fri Jul 12 11:41:02 2019 -0400

    15422: provenance and used_by endpoints traverse containers
    
    Fix up workbench rendering of provenance as well.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 514101244..de20b8858 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -176,7 +176,7 @@ class CollectionsController < ApplicationController
     if params["tab_pane"] == "Provenance_graph"
       @prov_svg = ProvenanceHelper::create_provenance_graph(@object.provenance, "provenance_svg",
                                                             {:request => request,
-                                                             :direction => :top_down,
+                                                             :direction => "RL",
                                                              :combine_jobs => :script_only}) rescue nil
     end
 
@@ -217,7 +217,7 @@ class CollectionsController < ApplicationController
         if params["tab_pane"] == "Used_by"
           @used_by_svg = ProvenanceHelper::create_provenance_graph(@object.used_by, "used_by_svg",
                                                                    {:request => request,
-                                                                    :direction => :top_down,
+                                                                    :direction => "LR",
                                                                     :combine_jobs => :script_only,
                                                                     :pdata_only => true}) rescue nil
         end
diff --git a/apps/workbench/app/controllers/container_requests_controller.rb b/apps/workbench/app/controllers/container_requests_controller.rb
index 385d9dc6d..587819dd4 100644
--- a/apps/workbench/app/controllers/container_requests_controller.rb
+++ b/apps/workbench/app/controllers/container_requests_controller.rb
@@ -11,7 +11,7 @@ class ContainerRequestsController < ApplicationController
   def generate_provenance(cr)
     return if params['tab_pane'] != "Provenance"
 
-    nodes = {cr[:uuid] => cr}
+    nodes = {}
     child_crs = []
     col_uuids = []
     col_pdhs = []
@@ -29,38 +29,33 @@ class ContainerRequestsController < ApplicationController
       end
     end
 
-    output_cols = {} # Indexed by UUID
-    input_cols = {} # Indexed by PDH
+    if nodes.length == 0
+      nodes[cr[:uuid]] = cr
+    end
+
+    pdh_to_col = {} # Indexed by PDH
     output_pdhs = []
 
     # Batch requests to get all related collections
     # First fetch output collections by UUID.
     Collection.filter([['uuid', 'in', col_uuids.uniq]]).each do |c|
-      output_cols[c[:uuid]] = c
       output_pdhs << c[:portable_data_hash]
+      pdh_to_col[c[:portable_data_hash]] = c
+      nodes[c[:uuid]] = c
     end
-    # Then, get only input collections by PDH. There could be more than one collection
-    # per PDH: the number of collections is used on the collection node label.
+    # Next, get input collections by PDH.
     Collection.filter(
       [['portable_data_hash', 'in', col_pdhs - output_pdhs]]).each do |c|
-      if input_cols[c[:portable_data_hash]]
-        input_cols[c[:portable_data_hash]] << c
-      else
-        input_cols[c[:portable_data_hash]] = [c]
-      end
+      nodes[c[:portable_data_hash]] = c
     end
 
     @svg = ProvenanceHelper::create_provenance_graph(
       nodes, "provenance_svg",
       {
         :request => request,
-        :direction => :top_down,
-        :output_collections => output_cols,
-        :input_collections => input_cols,
-        :cr_children_of => {
-          cr[:uuid] => child_crs.select{|child| child[:uuid]},
-        },
-      })
+        :pdh_to_uuid => pdh_to_col,
+      }
+    )
   end
 
   def show_pane_list
diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index 4f7bfcee5..bac1530d3 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -34,7 +34,6 @@ class JobsController < ApplicationController
 
     @svg = ProvenanceHelper::create_provenance_graph nodes, "provenance_svg", {
       :request => request,
-      :direction => :top_down,
       :all_script_parameters => true,
       :script_version_nodes => true}
   end
diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb
index c8863653a..6431057dd 100644
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@ -192,7 +192,6 @@ class PipelineInstancesController < ApplicationController
     if provenance
       @prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
         :request => request,
-        :direction => :top_down,
         :all_script_parameters => true,
         :combine_jobs => :script_and_version,
         :pips => pips,
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
index c06d83ae8..8a8337fe1 100644
--- a/apps/workbench/app/helpers/provenance_helper.rb
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -151,56 +151,97 @@ module ProvenanceHelper
       gr
     end
 
-    def cr_edges cr, edge_opts={}
+    # def cr_edges cr, edge_opts={}
+    #   gr = ""
+
+    #   gr += describe_node(cr[:uuid], {href: {controller: 'container_requests',
+    #                                          id: cr[:uuid]},
+    #                                   label: cr[:name],
+    #                                   shape: 'oval'})
+    #   # Connect child CRs
+    #   children = @opts[:cr_children_of].andand[cr[:uuid]]
+    #   if children
+    #     children.each do |child|
+    #       gr += edge(child[:uuid], cr[:uuid], {label: 'child'})
+    #     end
+    #   end
+    #   # Output collection node
+    #   if cr[:output_uuid] and @opts[:output_collections][cr[:output_uuid]]
+    #     c = @opts[:output_collections][cr[:output_uuid]]
+    #     gr += describe_node(c[:portable_data_hash],
+    #                         {
+    #                           label: c[:name],
+    #                           col_uuid: c[:uuid],
+    #                         })
+    #     gr += edge(cr[:uuid],
+    #                c[:portable_data_hash],
+    #                {label: 'output'})
+    #   end
+    #   # Input collection nodes
+    #   output_pdhs = @opts[:output_collections].values.collect{|oc|
+    #     oc[:portable_data_hash]}
+    #   ProvenanceHelper::cr_input_pdhs(cr).each do |pdh|
+    #     if not output_pdhs.include?(pdh)
+    #       # Search for collections on the same project first
+    #       cols = @opts[:input_collections][pdh].andand.select{|ic|
+    #         ic[:owner_uuid] == cr[:owner_uuid]}
+    #       if not cols or cols.empty?
+    #         # Search for any collection with this PDH
+    #         cols = @opts[:input_collections][pdh]
+    #       end
+    #       if cols
+    #         names = cols.collect{|x| x[:name]}.uniq
+    #       else
+    #         names = ['(collection not found)']
+    #       end
+    #       input_name = names.first
+    #       if names.length > 1
+    #         input_name += " + #{names.length - 1} more"
+    #       end
+    #       gr += describe_node(pdh, {label: input_name})
+    #     end
+    #     gr += edge(pdh, cr[:uuid], {label: 'input'})
+    #   end
+
+    #   gr
+    # end
+
+    def cr_edges cont, edge_opts={}
+      uuid = cont[:uuid]
       gr = ""
 
-      gr += describe_node(cr[:uuid], {href: {controller: 'container_requests',
-                                             id: cr[:uuid]},
-                                      label: cr[:name],
-                                      shape: 'oval'})
-      # Connect child CRs
-      children = @opts[:cr_children_of].andand[cr[:uuid]]
-      if children
-        children.each do |child|
-          gr += edge(child[:uuid], cr[:uuid], {label: 'child'})
+      gr += describe_node(cont[:uuid], {href: {controller: 'container_requests',
+                                             id: cont[:uuid]},
+                                        shape: 'oval',
+                                        label: cont[:name]})
+
+      ProvenanceHelper::find_collections cont[:mounts] do |collection_hash, collection_uuid, key|
+        if @opts[:pdh_to_uuid] and @opts[:pdh_to_uuid][collection_hash]
+          collection_uuid = @opts[:pdh_to_uuid][collection_hash].uuid
+          collection_hash = nil
+        end
+        if collection_uuid and @pdata[collection_uuid]
+          gr += describe_node(collection_uuid)
+          gr += edge(collection_uuid, uuid, {:label => key})
+        elsif collection_hash and @pdata[collection_hash]
+          gr += describe_node(collection_hash)
+          gr += edge(collection_hash, uuid, {:label => key})
         end
       end
-      # Output collection node
-      if cr[:output_uuid] and @opts[:output_collections][cr[:output_uuid]]
-        c = @opts[:output_collections][cr[:output_uuid]]
-        gr += describe_node(c[:portable_data_hash],
-                            {
-                              label: c[:name],
-                              col_uuid: c[:uuid],
-                            })
-        gr += edge(cr[:uuid],
-                   c[:portable_data_hash],
-                   {label: 'output'})
+
+      if cont[:container_image] and !@opts[:no_docker] and @pdata[cont[:container_image]]
+        gr += describe_node(cont[:container_image], {label: cont[:container_image]})
+        gr += edge(cont[:container_image], uuid, {label: "docker_image"})
       end
-      # Input collection nodes
-      output_pdhs = @opts[:output_collections].values.collect{|oc|
-        oc[:portable_data_hash]}
-      ProvenanceHelper::cr_input_pdhs(cr).each do |pdh|
-        if not output_pdhs.include?(pdh)
-          # Search for collections on the same project first
-          cols = @opts[:input_collections][pdh].andand.select{|ic|
-            ic[:owner_uuid] == cr[:owner_uuid]}
-          if not cols or cols.empty?
-            # Search for any collection with this PDH
-            cols = @opts[:input_collections][pdh]
-          end
-          if cols
-            names = cols.collect{|x| x[:name]}.uniq
-          else
-            names = ['(collection not found)']
-          end
-          input_name = names.first
-          if names.length > 1
-            input_name += " + #{names.length - 1} more"
-          end
-          gr += describe_node(pdh, {label: input_name})
-        end
-        gr += edge(pdh, cr[:uuid], {label: 'input'})
+
+      if cont[:output_uuid] and !edge_opts[:no_output] and @pdata[cont[:output_uuid]]
+        gr += describe_node(cont[:output_uuid])
+        gr += edge(uuid, cont[:output_uuid], {label: "output" })
+      end
+
+      if cont[:log_uuid] and !edge_opts[:no_log] and @pdata[cont[:log_uuid]]
+        gr += describe_node(cont[:log_uuid])
+        gr += edge(uuid, cont[:log_uuid], {label: "log"})
       end
 
       gr
@@ -364,11 +405,9 @@ module ProvenanceHelper
     gr = """strict digraph {
 node [fontsize=10,fontname=\"Helvetica,Arial,sans-serif\"];
 edge [fontsize=10,fontname=\"Helvetica,Arial,sans-serif\"];
-rankdir=RL;
 """
-
-    if opts[:direction] == :bottom_up
-      gr += "edge [dir=back];"
+    if ["LR", "RL"].include? opts[:direction]
+      gr += "rankdir=#{opts[:direction]};"
     end
 
     begin
diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 51a47f018..ede6fbd89 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -96,11 +96,11 @@ class Arvados::V1::CollectionsController < ApplicationController
   end
 
 
-  def find_collections(visited, sp, &b)
+  def find_collections(visited, sp, ignore_columns=[], &b)
     case sp
     when ArvadosModel
       sp.class.columns.each do |c|
-        find_collections(visited, sp[c.name.to_sym], &b) if c.name != "log"
+        find_collections(visited, sp[c.name.to_sym], &b) if !ignore_columns.include?(c.name)
       end
     when Hash
       sp.each do |k, v|
@@ -129,8 +129,6 @@ class Arvados::V1::CollectionsController < ApplicationController
       return if visited[loc.to_s]
     end
 
-    logger.debug "visiting #{uuid}"
-
     if loc
       # uuid is a portable_data_hash
       collections = Collection.readable_by(*@read_users).where(portable_data_hash: loc.to_s)
@@ -188,14 +186,11 @@ class Arvados::V1::CollectionsController < ApplicationController
           end
         end
 
-        Container.readable_by(*@read_users).where(["mounts like ?", "%#{loc.to_s}%"]).each do |c|
-          search_edges(visited, c.uuid, :search_down)
-        end
-
-        Container.readable_by(*@read_users).where(["container_image = '#{loc.to_s}'"]).each do |c|
-          search_edges(visited, c.uuid, :search_down)
+        Container.readable_by(*@read_users).where([Container.full_text_trgm + " like ?", "%#{loc.to_s}%"]).each do |c|
+          if c.output != loc.to_s && c.log != loc.to_s
+            search_edges(visited, c.uuid, :search_down)
+          end
         end
-
       end
     else
       # uuid is a regular Arvados UUID
@@ -215,7 +210,8 @@ class Arvados::V1::CollectionsController < ApplicationController
           end
         end
       elsif rsc == Container
-        Container.readable_by(*@read_users).where(uuid: uuid).each do |c|
+        c = Container.readable_by(*@read_users).where(uuid: uuid).limit(1).first
+        if c
           visited[uuid] = c.as_api_response
           if direction == :search_up
             # Follow upstream collections referenced in the script parameters
@@ -228,10 +224,37 @@ class Arvados::V1::CollectionsController < ApplicationController
             search_edges(visited, c.output, direction)
           end
         end
+      elsif rsc == ContainerRequest
+        c = ContainerRequest.readable_by(*@read_users).where(uuid: uuid).limit(1).first
+        if c
+          visited[uuid] = c.as_api_response
+          if direction == :search_up
+            # Follow upstream collections
+            find_collections(visited, c, ignore_columns=["log_uuid"]) do |hash, col_uuid|
+              search_edges(visited, hash, :search_up) if hash
+              search_edges(visited, col_uuid, :search_up) if col_uuid
+            end
+          elsif direction == :search_down
+            # Follow downstream job output
+            search_edges(visited, c.output_uuid, direction)
+          end
+        end
       elsif rsc == Collection
-        if c = Collection.readable_by(*@read_users).where(uuid: uuid).limit(1).first
-          search_edges(visited, c.portable_data_hash, direction)
-          visited[c.portable_data_hash] = c.as_api_response
+        c = Collection.readable_by(*@read_users).where(uuid: uuid).limit(1).first
+        if c
+          if direction == :search_up
+            visited[c.uuid] = c.as_api_response
+
+            ContainerRequest.readable_by(*@read_users).where(output_uuid: uuid).each do |cr|
+              search_edges(visited, cr.uuid, :search_up)
+            end
+
+            ContainerRequest.readable_by(*@read_users).where(log_uuid: uuid).each do |cr|
+              search_edges(visited, cr.uuid, :search_up)
+            end
+          elsif direction == :search_down
+            search_edges(visited, c.portable_data_hash, direction)
+          end
         end
       elsif rsc != nil
         rsc.where(uuid: uuid).each do |r|
@@ -261,15 +284,21 @@ class Arvados::V1::CollectionsController < ApplicationController
 
   def provenance
     visited = {}
-    search_edges(visited, @object[:portable_data_hash], :search_up)
-    search_edges(visited, @object[:uuid], :search_up)
+    if @object[:uuid]
+      search_edges(visited, @object[:uuid], :search_up)
+    else
+      search_edges(visited, @object[:portable_data_hash], :search_up)
+    end
     send_json visited
   end
 
   def used_by
     visited = {}
-    search_edges(visited, @object[:uuid], :search_down)
-    search_edges(visited, @object[:portable_data_hash], :search_down)
+    if @object[:uuid]
+      search_edges(visited, @object[:uuid], :search_down)
+    else
+      search_edges(visited, @object[:portable_data_hash], :search_down)
+    end
     send_json visited
   end
 

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list