[ARVADOS] updated: 2129d0fd84fa62a5498d9860f3307cd64dc2a704
git at public.curoverse.com
git at public.curoverse.com
Mon Feb 3 17:58:00 EST 2014
Summary of changes:
.../app/assets/javascripts/application.js | 7 +
.../app/assets/stylesheets/application.css | 3 +
apps/workbench/app/assets/stylesheets/loading.css | 374 +++++++++++++++++
.../app/controllers/application_controller.rb | 15 +-
.../controllers/pipeline_instances_controller.rb | 84 ++++-
apps/workbench/app/controllers/users_controller.rb | 6 +-
apps/workbench/app/helpers/application_helper.rb | 6 +-
.../app/helpers/pipeline_instances_helper.rb | 70 ++--
apps/workbench/app/models/arvados_base.rb | 4 +-
apps/workbench/app/models/node.rb | 2 +-
apps/workbench/app/models/pipeline_instance.rb | 4 +
apps/workbench/app/models/user.rb | 6 +-
apps/workbench/app/models/virtual_machine.rb | 2 +-
.../views/application/_arvados_attr_value.html.erb | 3 +-
.../app/views/application/_arvados_object.html.erb | 2 +-
.../application/_arvados_object_attr.html.erb | 5 +-
.../application/_breadcrumb_page_name.html.erb | 1 +
.../app/views/application/_breadcrumbs.html.erb | 20 +
.../app/views/application/_index.html.erb | 9 +-
.../app/views/application/_loading.html.erb | 190 +++++++++
.../workbench/app/views/application/destroy.js.erb | 3 +
apps/workbench/app/views/application/show.html.erb | 2 +-
apps/workbench/app/views/jobs/index.html.erb | 4 +-
apps/workbench/app/views/jobs/show.html.erb | 2 +-
.../app/views/layouts/application.html.erb | 11 +-
.../app/views/links/_breadcrumb_page_name.html.erb | 8 +
.../app/views/pipeline_instances/compare.html.erb | 75 ++++
.../app/views/pipeline_instances/index.html.erb | 31 ++-
.../app/views/pipeline_instances/show.html.erb | 101 ++++--
.../app/views/pipeline_instances/show.js.erb | 3 +
.../app/views/user_agreements/index.html.erb | 2 +
apps/workbench/app/views/users/_home.html.erb | 3 +-
apps/workbench/app/views/users/_tables.html.erb | 9 +-
apps/workbench/app/views/users/home.js.erb | 3 +
apps/workbench/app/views/users/index.html.erb | 62 ---
apps/workbench/app/views/users/welcome.html.erb | 2 +
apps/workbench/config/routes.rb | 4 +-
doc/api/schema/Link.textile | 2 +-
docker/build.sh | 42 ++
sdk/cli/Gemfile.lock | 2 +-
sdk/cli/test/test_arv-tag.rb | 10 +-
sdk/python/.gitignore | 1 +
sdk/python/arvados/__init__.py | 428 +-------------------
sdk/python/arvados/api.py | 91 +++++
sdk/python/arvados/collection.py | 6 +-
sdk/python/arvados/config.py | 34 ++
sdk/python/arvados/errors.py | 14 +
sdk/python/arvados/keep.py | 18 +-
sdk/python/arvados/stream.py | 7 +-
sdk/python/arvados/util.py | 306 ++++++++++++++
sdk/python/bin/arv-mount | 158 +++++++
sdk/python/requirements.txt | 1 +
sdk/python/setup.py | 4 +
sdk/python/test_util.py | 22 +
services/api/test/fixtures/jobs.yml | 22 +
.../functional/arvados/v1/jobs_controller_test.rb | 17 +
56 files changed, 1719 insertions(+), 604 deletions(-)
create mode 100644 apps/workbench/app/assets/stylesheets/loading.css
create mode 100644 apps/workbench/app/views/application/_breadcrumb_page_name.html.erb
create mode 100644 apps/workbench/app/views/application/_breadcrumbs.html.erb
create mode 100644 apps/workbench/app/views/application/_loading.html.erb
create mode 100644 apps/workbench/app/views/application/destroy.js.erb
create mode 100644 apps/workbench/app/views/links/_breadcrumb_page_name.html.erb
create mode 100644 apps/workbench/app/views/pipeline_instances/compare.html.erb
create mode 100644 apps/workbench/app/views/pipeline_instances/show.js.erb
create mode 100644 apps/workbench/app/views/users/home.js.erb
delete mode 100644 apps/workbench/app/views/users/index.html.erb
create mode 100644 docker/build.sh
create mode 100644 sdk/python/arvados/api.py
create mode 100644 sdk/python/arvados/config.py
create mode 100644 sdk/python/arvados/errors.py
create mode 100644 sdk/python/arvados/util.py
create mode 100755 sdk/python/bin/arv-mount
create mode 100644 sdk/python/test_util.py
via 2129d0fd84fa62a5498d9860f3307cd64dc2a704 (commit)
via d8c56aa6534d87105201372cd5f9f965a8246ea4 (commit)
via 6038a018b758e1a4babc5669df50622cd470df2f (commit)
via bfa69a748122aa2515c85052c8e38752d290ecc1 (commit)
via 9899ee7ffeaa915e36df61cd814cf6e06db7ebe9 (commit)
via 5dd1249f3ec74643957202977f92e42be78f65d8 (commit)
via 13316cbdecd85557ade334262f7cbfec22c4b275 (commit)
via 36cb102dca2aba54c01b4a4d08bb7c2a2d4ad337 (commit)
via 19552d4ecd7552ab5dfdfc6d894b22dce6b7f107 (commit)
via 18c60ea458cb2a97c9d7318e702da8296e102466 (commit)
via 507442f6712d751b947083fa6946ee6d96f8ad96 (commit)
via ce408713e3067fc278a5a8f9c2d534567f65c636 (commit)
via 4d779096453fd54437df2fdafd682b550e24861f (commit)
via 1790ceeb9a977c6aeae9c1a5d8b6dc489bf27275 (commit)
via b2bc35d29fe37bf82f314b48880adba51ae1e343 (commit)
via 7330db33df73dbb204d362253668436cb5caa7b3 (commit)
via f222423ba06019e796e5024d300398577e9987e1 (commit)
via dfca93f7d608930350b8aa680a3ffb3d35d63306 (commit)
via 9fd96762299e182afbf69e76e0560c748591af47 (commit)
via 4e92e5890ae7b44545052db029863e886a36cc79 (commit)
via 2ecdb027a8866550d923c702fbbef888b76e6702 (commit)
via 9b78ee7e66152bda07daf881d1f9305a11100ce6 (commit)
via 4e501ab0ba22d9d34d478d32638d53e99ecb9a6c (commit)
via 4f9c8ded1490b6b2376b426a131d403f817e7549 (commit)
via e4ef3bbeb9d89946ee88aa031204e76156bb9d57 (commit)
via 5de6e729142a497ecb6e6dc40e2239cae07cd18d (commit)
via 013efe3cfb6e53b372be1f785646e76a7965a948 (commit)
via 9ea1a87fc8e9725473e909f543af2b83b03237a4 (commit)
via 3a342f5de4da4b155d551e54aeeab51eed9f8d68 (commit)
via 1440352641349c15cc98fea5bf69e0a8b40d7a0c (commit)
via 6ae89f63b495395880b34645c84c813cbfacee78 (commit)
via eaa436ddd4aadf4fad387ea6115e35c885828ce0 (commit)
via 043f54270bb129da9f5f07c2c975c6e89edd4a22 (commit)
from 5a016cb9182b979b94df49c8b20856853541396e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2129d0fd84fa62a5498d9860f3307cd64dc2a704
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Feb 3 14:52:10 2014 -0800
Hide this/other/both legend if only one pipeline_instance is shown.
diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb
index 1bb95e0..a42a734 100644
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@ -4,11 +4,11 @@ class PipelineInstancesController < ApplicationController
include PipelineInstancesHelper
def show
- pipelines = [@object]
+ @pipelines = [@object]
if params[:compare]
PipelineInstance.where(uuid: params[:compare]).each do |p|
- pipelines << p
+ @pipelines << p
end
end
@@ -17,7 +17,7 @@ class PipelineInstancesController < ApplicationController
pips = {}
n = 1
- pipelines.each do |p|
+ @pipelines.each do |p|
collections = []
p.components.each do |k, v|
diff --git a/apps/workbench/app/views/pipeline_instances/show.html.erb b/apps/workbench/app/views/pipeline_instances/show.html.erb
index 84c1e30..cf56b97 100644
--- a/apps/workbench/app/views/pipeline_instances/show.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/show.html.erb
@@ -65,11 +65,13 @@ table.pipeline-components-table div.progress {
</tfoot>
</table>
-<div style="text-align: center">
-<span class="pipeline_color_legend" style="background: #88ff88">This pipeline</span>
-<span class="pipeline_color_legend" style="background: #8888ff">Comparison pipeline</span>
-<span class="pipeline_color_legend" style="background: #88ffff">Shared by both pipelines</span>
-</div>
+<% if @pipelines.count > 1 %>
+ <div style="text-align: center">
+ <span class="pipeline_color_legend" style="background: #88ff88">This pipeline</span>
+ <span class="pipeline_color_legend" style="background: #8888ff">Comparison pipeline</span>
+ <span class="pipeline_color_legend" style="background: #88ffff">Shared by both pipelines</span>
+ </div>
+<% end %>
<%= render partial: 'application/svg_div', locals: {
divId: "provenance_graph",
commit d8c56aa6534d87105201372cd5f9f965a8246ea4
Merge: 6038a01 19552d4
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Feb 3 14:56:30 2014 -0800
Merge branch '1976-pipeline-progress'
Conflicts:
apps/workbench/app/views/pipeline_instances/show.html.erb
diff --cc apps/workbench/app/views/pipeline_instances/show.html.erb
index 96c32db,9000750..84c1e30
--- a/apps/workbench/app/views/pipeline_instances/show.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/show.html.erb
@@@ -1,52 -1,72 +1,87 @@@
<% content_for :css do %>
+ .pipeline_color_legend {
+ padding-left: 1em;
+ padding-right: 1em;
+ }
+ table.pipeline-components-table thead th {
+ text-align: bottom;
+ }
+ table.pipeline-components-table div.progress {
+ margin-bottom: 0;
+ }
<% end %>
+ <br />
- <table class="table table-condensed table-hover topalign">
+ <table class="table pipeline-components-table">
+ <colgroup>
+ <col width="15%" />
+ <col width="15%" />
+ <col width="35%" />
+ <col width="35%" />
+ </colgroup>
<thead>
+ <tr>
+ <th>
+ component
+ </th><th>
+ progress
+ <%= link_to '(refresh)', request.fullpath, class: 'refresh', remote: true, method: 'get' %>
+ </th><th>
+ script, version
+ </th><th>
+ output
+ </th>
+ </tr>
</thead>
<tbody>
- <% @object.attributes_for_display.each do |attr, attrvalue| %>
- <% if attr == 'components' and attrvalue.is_a? Hash %>
-
- <tr class="info"><td><%= attr %></td><td>
- <table class="table">
- <% render_pipeline_jobs.each do |pj| %>
- <tr><% %w(index name result job_link script script_version progress_detail progress_bar output_link).each do |key| %>
- <td>
- <% if key == 'script_version' %>
- <%= pj[key.to_sym][0..6] rescue '' %>
- <% else %>
- <%= pj[key.to_sym] %>
- <% end %>
- </td>
- <% end %>
- </tr>
- <% end %>
- </table>
- </td></tr>
-
- <% else %>
- <%= render partial: 'application/arvados_object_attr', locals: { attr: attr, attrvalue: attrvalue } %>
- <% end %>
+ <% render_pipeline_jobs.each do |pj| %>
+ <tr>
+ <td>
+ <% if pj[:job].andand[:uuid] %>
+ <%= link_to pj[:name], job_url(id: pj[:job][:uuid]) %>
+ <% else %>
+ <%= pj[:name] %>
+ <% end %>
+ </td><td>
+ <%= pj[:progress_bar] %>
+ <% if pj[:job].andand[:cancelled_at] %>
+ <span class="pull-right label label-warning">cancelled</span>
+ <% elsif pj[:failed] %>
+ <span class="pull-right label label-warning">failed</span>
+ <% elsif pj[:result] == 'queued' %>
+ <span class="pull-right label">queued</span>
+ <% end %>
+ </td><td>
+ <%= pj[:script] %>
+ <br /><span class="deemphasize"><%= pj[:script_version] %></span>
+ </td><td>
+ <%= link_to_if_arvados_object pj[:output] %>
+ </td>
+ </tr>
<% end %>
</tbody>
+ <tfoot>
+ <tr><td colspan="4"></td></tr>
+ </tfoot>
</table>
+<div style="text-align: center">
+<span class="pipeline_color_legend" style="background: #88ff88">This pipeline</span>
+<span class="pipeline_color_legend" style="background: #8888ff">Comparison pipeline</span>
+<span class="pipeline_color_legend" style="background: #88ffff">Shared by both pipelines</span>
+</div>
+
+ <%= render partial: 'application/svg_div', locals: {
+ divId: "provenance_graph",
+ svgId: "provenance_svg",
+ svg: @prov_svg } %>
+
- <pre>
- <%= JSON.pretty_generate(@object.attributes) rescue nil %>
- </pre>
+ <div style="height: 1em"></div>
+
+ <%= render partial: 'arvados_object' %>
+
+ <% if @object.active %>
+ <% content_for :js do %>
+ setInterval(function(){$('a.refresh').click()}, 30000);
+ <% end %>
+ <% end %>
commit 6038a018b758e1a4babc5669df50622cd470df2f
Merge: 5a016cb bfa69a7
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Feb 3 14:44:19 2014 -0800
Merge branch 'master' into 1977-provenance-report
Conflicts:
apps/workbench/app/controllers/pipeline_instances_controller.rb
apps/workbench/app/views/jobs/show.html.erb
diff --cc apps/workbench/app/controllers/pipeline_instances_controller.rb
index 55ee02b,d776941..1bb95e0
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@@ -1,56 -1,77 +1,130 @@@
class PipelineInstancesController < ApplicationController
+ skip_before_filter :find_object_by_uuid, only: :compare
+ before_filter :find_objects_by_uuid, only: :compare
+ include PipelineInstancesHelper
+ def show
+ pipelines = [@object]
+
+ if params[:compare]
- PipelineInstance.where(uuid: params[:compare]).each do |p| pipelines << p end
++ PipelineInstance.where(uuid: params[:compare]).each do |p|
++ pipelines << p
++ end
+ end
+
+ count = {}
+ provenance = {}
+ pips = {}
+ n = 1
+
+ pipelines.each do |p|
+ collections = []
+
+ p.components.each do |k, v|
+ j = v[:job]
+
+ uuid = j[:uuid].intern
+ provenance[uuid] = j
+ pips[uuid] = 0 unless pips[uuid] != nil
+ pips[uuid] |= n
+
+ collections << j[:output]
+ ProvenanceHelper::find_collections(j[:script_parameters]).each do |k|
+ collections << k
+ end
+
+ uuid = j[:script_version].intern
+ provenance[uuid] = {:uuid => uuid}
+ pips[uuid] = 0 unless pips[uuid] != nil
+ pips[uuid] |= n
+ end
+
+ Collection.where(uuid: collections).each do |c|
+ uuid = c.uuid.intern
+ provenance[uuid] = c
+ pips[uuid] = 0 unless pips[uuid] != nil
+ pips[uuid] |= n
+ end
+
+ n = n << 1
+ end
+
- #puts pips
-
+ @prov_svg = ProvenanceHelper::create_provenance_graph provenance, "provenance_svg", {
+ :all_script_parameters => true,
+ :combine_jobs => :script_and_version,
+ :script_version_nodes => true,
+ :pips => pips }
+ end
+
+ def compare
+ @breadcrumb_page_name = 'compare'
+
+ @rows = [] # each is {name: S, components: [...]}
+
+ # Build a table: x=pipeline y=component
+ @objects.each_with_index do |pi, pi_index|
+ pipeline_jobs(pi).each do |component|
+ # Find a cell with the same name as this component but no
+ # entry for this pipeline
+ target_row = nil
+ @rows.each_with_index do |row, row_index|
+ if row[:name] == component[:name] and !row[:components][pi_index]
+ target_row = row
+ end
+ end
+ if !target_row
+ target_row = {name: component[:name], components: []}
+ @rows << target_row
+ end
+ target_row[:components][pi_index] = component
+ end
+ end
+
+ @rows.each do |row|
+ # Build a "normal" pseudo-component for this row by picking the
+ # most common value for each attribute. If all values are
+ # equally common, there is no "normal".
+ normal = {} # attr => most common value
+ highscore = {} # attr => how common "normal" is
+ score = {} # attr => { value => how common }
+ row[:components].each do |pj|
+ pj.each do |k,v|
+ vstr = for_comparison v
+ score[k] ||= {}
+ score[k][vstr] = (score[k][vstr.to_s] || 0) + 1
+ highscore[k] ||= 0
+ if score[k][vstr] == highscore[k]
+ # tie for first place = no "normal"
+ normal.delete k
+ elsif score[k][vstr] == highscore[k] + 1
+ # more pipelines have v than anything else
+ highscore[k] = score[k][vstr]
+ normal[k] = vstr
+ end
+ end
+ end
+
+ # Add a hash in component[:is_normal]: { attr => is_the_value_normal? }
+ row[:components].each do |pj|
+ pj[:is_normal] = {}
+ pj.each do |k,v|
+ pj[:is_normal][k] = (normal.has_key?(k) && normal[k] == for_comparison(v))
+ end
+ end
+ end
+ end
+
+ protected
+ def for_comparison v
+ if v.is_a? Hash or v.is_a? Array
+ v.to_json
+ else
+ v.to_s
+ end
+ end
+
+ def find_objects_by_uuid
+ @objects = model_class.where(uuid: params[:uuids])
+ end
+
end
diff --cc apps/workbench/app/views/jobs/show.html.erb
index 5022b8b,0000000..2bc74ca
mode 100644,000000..100644
--- a/apps/workbench/app/views/jobs/show.html.erb
+++ b/apps/workbench/app/views/jobs/show.html.erb
@@@ -1,6 -1,0 +1,6 @@@
- <%= render :partial => 'application/arvados_object' %>
++<%= render partial: 'arvados_object' %>
+
+<%= render partial: 'application/svg_div', locals: {
+ divId: "provenance_graph",
+ svgId: "provenance_svg",
+ svg: @svg } %>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list