[ARVADOS] updated: 04bb94072486e32ed9784be3a22c43d961c84bee
git at public.curoverse.com
git at public.curoverse.com
Tue Jan 28 16:23:15 EST 2014
Summary of changes:
apps/admin/setup-new-user.rb | 2 +-
.../app/assets/javascripts/keep_disks.js.coffee | 0
.../app/assets/stylesheets/keep_disks.css.scss | 0
.../app/controllers/collections_controller.rb | 107 +++++-----
.../app/controllers/keep_disks_controller.rb | 2 +
.../workbench}/app/helpers/keep_disks_helper.rb | 0
apps/workbench/app/models/arvados_api_client.rb | 7 +
apps/workbench/app/models/arvados_base.rb | 32 ++--
apps/workbench/app/models/collection.rb | 4 +
apps/workbench/app/models/keep_disk.rb | 5 +
apps/workbench/app/models/node.rb | 4 +-
.../app/views/layouts/application.html.erb | 26 ++-
apps/workbench/config/routes.rb | 1 +
.../test/functional/keep_disks_controller_test.rb | 4 +
.../test/unit/helpers/keep_disks_helper_test.rb | 0
apps/workbench/test/unit/keep_disk_test.rb | 4 +
doc/_includes/navbar_left.html | 2 +-
doc/_includes/run-md5sum.py | 2 +-
doc/admin/cheat_sheet.textile | 2 +-
doc/api/resources.textile | 2 +-
.../tutorials/running-external-program.textile | 6 +-
sdk/cli/Gemfile | 2 +
sdk/cli/Gemfile.lock | 36 ++--
sdk/cli/arvados-cli.gemspec | 2 +-
sdk/cli/bin/arv | 2 +-
sdk/cli/test/test_arv-collection-create.rb | 38 +++
sdk/ruby/Gemfile | 4 +
sdk/{cli => ruby}/Gemfile.lock | 46 ++---
sdk/{cli => ruby}/Rakefile | 0
sdk/ruby/arvados.gemspec | 3 +-
sdk/ruby/lib/arvados.rb | 68 ++++--
sdk/ruby/test/test_big_request.rb | 38 +++
.../api/app/controllers/application_controller.rb | 39 ++--
.../arvados/v1/collections_controller.rb | 28 ++-
.../app/controllers/arvados/v1/jobs_controller.rb | 1 +
.../arvados/v1/keep_disks_controller.rb | 35 ++--
.../app/controllers/arvados/v1/nodes_controller.rb | 1 +
.../controllers/arvados/v1/schema_controller.rb | 21 +--
.../arvados/v1/user_agreements_controller.rb | 3 +-
.../app/controllers/arvados/v1/users_controller.rb | 7 +-
.../arvados/v1/virtual_machines_controller.rb | 1 +
services/api/app/controllers/static_controller.rb | 2 +
.../app/controllers/user_sessions_controller.rb | 1 +
services/api/app/models/arvados_model.rb | 27 +++
services/api/app/models/node.rb | 20 ++-
services/api/config/environments/test.rb.example | 1 +
services/api/config/routes.rb | 1 +
.../20140124222114_fix_link_kind_underscores.rb | 17 ++
services/api/db/schema.rb | 2 +-
services/api/lib/kind_and_etag.rb | 2 +-
services/api/test/fixtures/groups.yml | 6 +
services/api/test/fixtures/links.yml | 32 +++
services/api/test/fixtures/virtual_machines.yml | 4 +
.../arvados/v1/groups_controller_test.rb | 8 +-
.../arvados/v1/keep_disks_controller_test.rb | 30 ++--
.../functional/arvados/v1/links_controller_test.rb | 21 ++
.../functional/arvados/v1/users_controller_test.rb | 2 +-
.../api/test/integration/collections_api_test.rb | 21 ++-
services/keep/INSTALL | 20 --
services/keep/keep.rb | 241 --------------------
60 files changed, 532 insertions(+), 513 deletions(-)
copy {services/api => apps/workbench}/app/assets/javascripts/keep_disks.js.coffee (100%)
copy {services/api => apps/workbench}/app/assets/stylesheets/keep_disks.css.scss (100%)
create mode 100644 apps/workbench/app/controllers/keep_disks_controller.rb
copy {services/api => apps/workbench}/app/helpers/keep_disks_helper.rb (100%)
create mode 100644 apps/workbench/app/models/keep_disk.rb
create mode 100644 apps/workbench/test/functional/keep_disks_controller_test.rb
copy {services/api => apps/workbench}/test/unit/helpers/keep_disks_helper_test.rb (100%)
create mode 100644 apps/workbench/test/unit/keep_disk_test.rb
create mode 100644 sdk/cli/test/test_arv-collection-create.rb
create mode 100644 sdk/ruby/Gemfile
copy sdk/{cli => ruby}/Gemfile.lock (54%)
copy sdk/{cli => ruby}/Rakefile (100%)
create mode 100644 sdk/ruby/test/test_big_request.rb
create mode 100644 services/api/db/migrate/20140124222114_fix_link_kind_underscores.rb
create mode 100644 services/api/test/fixtures/virtual_machines.yml
delete mode 100644 services/keep/INSTALL
delete mode 100755 services/keep/keep.rb
via 04bb94072486e32ed9784be3a22c43d961c84bee (commit)
via 4d0a43a4c6e7567f1ff6b5014ccabca5cf111b83 (commit)
via d1e128422a34c9ab706eec0225d4a5f6b78760ad (commit)
via e17169e05c2e69d5bb46792bc33ef3c92f9c5237 (commit)
via 19d0b4c509ec720f9ffc1ea13f758c5825308834 (commit)
via 6f4a1bcdf7fcddcad49769adecb1f9a4c459b822 (commit)
via 8506c7afd5dc27d8fe40d10d53283cc7af9647e7 (commit)
via 7c99060e90f027b77c82fd8c66902095197261af (commit)
via 0332c1678b9d7b500f0b11ac60ef387d85e5097c (commit)
via 0693d6260764bff4deeb12fcd14b8906de9b83ae (commit)
via 1d84f4602ec9ef13689be6acfb27bc98e6dc367a (commit)
via ee08ff24e0126f602bfd9f7371c90f75801c3597 (commit)
via d6e0ee35e9aa71891feacefcc65e98ba8a7615e5 (commit)
via c0633e110d4920794e1de39c4e98cf95fee670cf (commit)
via 5e2a966ce8c7e499691f955ba5b5b37a5473e325 (commit)
via 5f5bb630ce1830f1665edbe3c90db2254debd70c (commit)
via ef1553ac09f9fd5afa7125f0c423139d112b7913 (commit)
via 01567e1c9675b9342187ae5872dbd2fd271fd838 (commit)
via bdf067b92a5f0659a3c212c136a9cb9882f5924d (commit)
via 453593b30c7e0fd59fe25d62772113103c86811b (commit)
via 50a594a71b26bd091731ae52bcfed5ddd27a0c9f (commit)
via 2a4f5a6e96c19ea65c35af705ab2561cccc2289a (commit)
via f66156884166e4f52327b744d1a8303f87b8ac63 (commit)
via f902ff3c640e3d7d279cb0ea112643e405c871f3 (commit)
via fa8ab6f3576881b80e2edf4b9066ca15f46f05b4 (commit)
via 79548cf316986bb06dde783dbdfea9c5584ac886 (commit)
via 92ccdadd52d417f5aa4494edaee0d2cec4cb3d43 (commit)
via 4cda46df7b9c5493bfc49af70e5b27d66271a873 (commit)
via 23aec28ff071904f5145ebb535a1c6d32e915f85 (commit)
via 28590c1712c7c3c552cb5acbf01c482dc834b927 (commit)
via 59994ce0b7623c84d2d633c7c0bac0966245932c (commit)
via 6779e6dd8bd666a5f398345e88c077dccca901ea (commit)
via 9698348290b7cdf762eecc6d3d365657e2930f14 (commit)
via 746240ca83082eb1066a1bfd35823c71708dc6ec (commit)
via b509e32fbef0ff346c26669a22b87576390285ba (commit)
from ba9bab4e6d0817a1eec7ecfed7f03a03e6f94d5a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 04bb94072486e32ed9784be3a22c43d961c84bee
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jan 28 16:24:01 2014 -0500
Workbench is able to put together a graph from the provenance document served by the API server. Needs more testing.
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 2508d3d..57b9bab 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -52,27 +52,28 @@ class CollectionsController < ApplicationController
self.response_body = FileStreamer.new opts
end
- def describe_node(uuid)
+ def self.describe_node(uuid)
+ uuid = uuid.to_s
rsc = ArvadosBase::resource_class_for_uuid uuid
if rsc
- "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{url_for rsc}/#{uuid}\"];"
+ "\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"/#{rsc.to_s.underscore.pluralize rsc}/#{uuid}\"];"
else
""
end
end
- def describe_script(job)
- #"""\"#{job.script_version}\" [label=\"#{job.script}: #{job.script_version}\"];
+ def self.describe_script(job)
+ # """\"#{job.script_version}\" [label=\"#{job.script}: #{job.script_version}\"];
# \"#{job.uuid}\" -> \"#{job.script_version}\" [label=\"script\"];"""
"\"#{job.uuid}\" [label=\"#{job.script}\\n#{job.script_version}\"];"
end
- def job_uuid(job)
+ def self.job_uuid(job)
"#{job.script}\\n#{job.script_version}"
end
- def collection_uuid(uuid)
- m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid)
+ def self.collection_uuid(uuid)
+ m = /([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?/.match(uuid.to_s)
if m
m[1]
else
@@ -80,7 +81,7 @@ class CollectionsController < ApplicationController
end
end
- def script_param_edges(visited, job, prefix, sp)
+ def self.script_param_edges(visited, job, prefix, sp)
gr = ""
if sp and not sp.empty?
case sp
@@ -89,89 +90,93 @@ class CollectionsController < ApplicationController
if prefix.size > 0
k = prefix + "::" + k.to_s
end
- gr += script_param_edges(visited, job, k.to_s, v)
+ gr += CollectionsController::script_param_edges(visited, job, k.to_s, v)
end
when Array
sp.each do |v|
- gr += script_param_edges(visited, job, prefix, v)
+ gr += CollectionsController::script_param_edges(visited, job, prefix, v)
end
else
m = collection_uuid(sp)
if m
gr += "\"#{job_uuid(job)}\" -> \"#{m}\" [label=\" #{prefix}\"];"
- gr += generate_provenance_edges(visited, m)
+ gr += CollectionsController::generate_provenance_edges(visited, m)
end
end
end
gr
end
- def generate_provenance_edges(visited, uuid)
+ def self.generate_provenance_edges(pdata, uuid)
gr = ""
- m = collection_uuid(uuid)
+ m = CollectionsController::collection_uuid(uuid)
+ uuid = m if m
- if not uuid or uuid.empty? or visited[uuid] or visited[m]
+ uuid = uuid.intern if uuid
+
+ if (not uuid) or uuid.empty? \
+ or (pdata[uuid] and pdata[uuid][:_visited]) \
+ or (not pdata[uuid])
+
+ puts "already visited #{uuid}"
return ""
end
- #puts "visiting #{uuid}"
+ puts "visiting #{uuid}"
+
+ pdata[uuid][:_visited] = true
if m
# uuid is a collection
- uuid = m
- visited[uuid] = true
-
- gr += describe_node(uuid)
+ gr += CollectionsController::describe_node(uuid)
- Job.where(output: uuid).each do |job|
- #gr += describe_node(job_uuid(job))
- gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\" output\"];"
- gr += generate_provenance_edges(visited, job.uuid)
- end
-
- Job.where(log: uuid).each do |job|
- #gr += describe_node(job_uuid(job))
- gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\" log\"];"
- gr += generate_provenance_edges(visited, job.uuid)
+ pdata.each do |k, job|
+ if job[:output] == uuid
+ gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"output\"];"
+ gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
+ end
+ if job[:log] == uuid
+ gr += "\"#{uuid}\" -> \"#{job_uuid(job)}\" [label=\"log\"];"
+ gr += CollectionsController::generate_provenance_edges(pdata, job[:uuid])
+ end
end
-
else
- visited[uuid] = true
-
# uuid is something else
rsc = ArvadosBase::resource_class_for_uuid uuid
if rsc == Job
- Job.where(uuid: uuid).each do |job|
- gr += script_param_edges(visited, job, "", job.script_parameters)
- #gr += describe_script(job)
+ job = pdata[uuid]
+ if job
+ gr += CollectionsController::script_param_edges(pdata, job, "", job.script_parameters)
end
else
- gr += describe_node(uuid)
+ gr += CollectionsController::describe_node(uuid)
end
end
- Link.where(head_uuid: uuid, link_class: "provenance").each do |link|
- gr += describe_node(link.tail_uuid)
- gr += "\"#{link.head_uuid}\" -> \"#{link.tail_uuid}\" [label=\" #{link.name}\", href=\"/links/#{link.uuid}\"];"
- gr += generate_provenance_edges(visited, link.tail_uuid)
+ pdata.each do |k, link|
+ if link[:head_uuid] == uuid.to_s and link[:link_class] == "provenance"
+ gr += CollectionsController::describe_node(link[:tail_uuid])
+ gr += "\"#{link[:head_uuid]}\" -> \"#{link[:tail_uuid]}\" [label=\" #{link[:name]}\", href=\"/links/#{link[:uuid]}\"];"
+ gr += CollectionsController::generate_provenance_edges(pdata, link[:tail_uuid])
+ end
end
- #puts "finished #{uuid}"
+ puts "finished #{uuid}"
gr
end
- def create_provenance_graph(uuid)
+ def self.create_provenance_graph(pdata, uuid)
require 'open3'
gr = """strict digraph {
-//rankdir=LR;
node [fontsize=8,shape=box];
edge [dir=back,fontsize=8];"""
- visited = {}
- gr += generate_provenance_edges(visited, uuid)
+ puts "pdata is #{pdata}"
+
+ gr += CollectionsController::generate_provenance_edges(pdata, uuid)
gr += "}"
svg = ""
@@ -232,8 +237,10 @@ edge [dir=back,fontsize=8];"""
@sourcedata[collection.uuid][:collection] = collection
end
end
-
- @prov_svg = create_provenance_graph(@object.uuid)
+
+ Collection.where(uuid: @object.uuid).each do |u|
+ @prov_svg = CollectionsController::create_provenance_graph u.provenance, u.uuid
+ end
end
protected
diff --git a/apps/workbench/app/models/collection.rb b/apps/workbench/app/models/collection.rb
index cfbb640..94a2b95 100644
--- a/apps/workbench/app/models/collection.rb
+++ b/apps/workbench/app/models/collection.rb
@@ -12,4 +12,8 @@ class Collection < ArvadosBase
def attribute_editable?(attr)
false
end
+
+ def provenance
+ $arvados_api_client.api "collections/#{self.uuid}/", "provenance"
+ end
end
commit 4d0a43a4c6e7567f1ff6b5014ccabca5cf111b83
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jan 28 14:06:40 2014 -0500
API server /collections/uuid/provenance endpoint works.
diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 918d499..48544a2 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -92,8 +92,8 @@ class Arvados::V1::CollectionsController < ApplicationController
if m
# uuid is a collection
uuid = m
- Collection.where(uuid:"uuid").each do |c|
- visited[uuid] = c
+ Collection.where(uuid: uuid).each do |c|
+ visited[uuid] = c.as_api_response
end
Job.where(output: uuid).each do |job|
@@ -105,32 +105,32 @@ class Arvados::V1::CollectionsController < ApplicationController
end
else
- visited[uuid] = true
-
# uuid is something else
- rsc = ArvadosBase::resource_class_for_uuid uuid
-
+ rsc = ArvadosModel::resource_class_for_uuid uuid
if rsc == Job
Job.where(uuid: uuid).each do |job|
- visited[uuid] = job
+ visited[uuid] = job.as_api_response
script_param_edges(visited, job, "", job.script_parameters)
end
+ elsif rsc != nil
+ rsc.where(uuid: uuid).each do |r|
+ visited[uuid] = r.as_api_response
+ end
end
end
Link.where(head_uuid: uuid, link_class: "provenance").each do |link|
+ visited[link.uuid] = link.as_api_response
generate_provenance_edges(visited, link.tail_uuid)
end
#puts "finished #{uuid}"
-
- gr
end
- def provenance(id)
+ def provenance
visited = {}
- generate_provenance_edges(visited, id[:uuid])
- visited
+ generate_provenance_edges(visited, @object[:uuid])
+ render json: visited
end
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index 2999b52..69eae92 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -1,3 +1,4 @@
+require 'assign_uuid'
class ArvadosModel < ActiveRecord::Base
self.abstract_class = true
@@ -140,4 +141,30 @@ class ArvadosModel < ActiveRecord::Base
end
end
end
+
+ def self.resource_class_for_uuid(uuid)
+ if uuid.is_a? ArvadosModel
+ return uuid.class
+ end
+ unless uuid.is_a? String
+ return nil
+ end
+ if uuid.match /^[0-9a-f]{32}(\+[^,]+)*(,[0-9a-f]{32}(\+[^,]+)*)*$/
+ return Collection
+ end
+ resource_class = nil
+
+ Rails.application.eager_load!
+ uuid.match /^[0-9a-z]{5}-([0-9a-z]{5})-[0-9a-z]{15}$/ do |re|
+ ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
+ if k.respond_to?(:uuid_prefix)
+ if k.uuid_prefix == re[1]
+ return k
+ end
+ end
+ end
+ end
+ nil
+ end
+
end
commit d1e128422a34c9ab706eec0225d4a5f6b78760ad
Merge: 19d0b4c e17169e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jan 28 11:10:47 2014 -0500
Merge commit 'e17169e05c2e69d5bb46792bc33ef3c92f9c5237' into 1977-provenance-report
commit 19d0b4c509ec720f9ffc1ea13f758c5825308834
Merge: 6f4a1bc 8506c7a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jan 28 10:47:07 2014 -0500
Merge branch 'master' into 1977-provenance-report
commit 6f4a1bcdf7fcddcad49769adecb1f9a4c459b822
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jan 28 10:45:54 2014 -0500
Work in progress
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index bf7dde0..2508d3d 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -1,11 +1,7 @@
class CollectionsController < ApplicationController
- skip_before_filter :find_object_by_uuid, :only => [:graph]
+ skip_before_filter :find_object_by_uuid, :only => [:provenance]
skip_before_filter :check_user_agreements, :only => [:show_file]
- def graph
- index
- end
-
def index
if params[:search].andand.length.andand > 0
tags = Link.where(any: ['contains', params[:search]])
diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index d14de73..918d499 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -115,6 +115,7 @@ class Arvados::V1::CollectionsController < ApplicationController
visited[uuid] = job
script_param_edges(visited, job, "", job.script_parameters)
end
+ end
end
Link.where(head_uuid: uuid, link_class: "provenance").each do |link|
@@ -126,14 +127,14 @@ class Arvados::V1::CollectionsController < ApplicationController
gr
end
- def provenance
+ def provenance(id)
visited = {}
- generate_provenance_edges(visited, @object.uuid)
+ generate_provenance_edges(visited, id[:uuid])
visited
end
- protected
+ protected
def find_object_by_uuid
super
if !@object and !params[:uuid].match(/^[0-9a-f]+\+\d+$/)
@@ -152,4 +153,5 @@ class Arvados::V1::CollectionsController < ApplicationController
end
end
end
+
end
diff --git a/services/api/config/routes.rb b/services/api/config/routes.rb
index 65b6a17..e837e38 100644
--- a/services/api/config/routes.rb
+++ b/services/api/config/routes.rb
@@ -96,6 +96,7 @@ Server::Application.routes.draw do
match '/repositories/get_all_permissions' => 'repositories#get_all_permissions'
get '/user_agreements/signatures' => 'user_agreements#signatures'
post '/user_agreements/sign' => 'user_agreements#sign'
+ get '/collections/:uuid/provenance' => 'collections#provenance'
resources :collections
resources :links
resources :nodes
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list