[ARVADOS] created: 38b9c64d051ebf0d1e430c498249caf608dcce2f
git at public.curoverse.com
git at public.curoverse.com
Fri Feb 28 16:41:03 EST 2014
at 38b9c64d051ebf0d1e430c498249caf608dcce2f (commit)
commit 38b9c64d051ebf0d1e430c498249caf608dcce2f
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 28 16:36:44 2014 -0500
Finished code to assemble a new manifest from the supplied files and collections.
New collection is added to API server and to Keep.
Added --extract command to arv-normalize to produce manifest for a single file in a collection.
diff --git a/apps/workbench/app/controllers/actions_controller.rb b/apps/workbench/app/controllers/actions_controller.rb
index 94bfd1e..6f5ed60 100644
--- a/apps/workbench/app/controllers/actions_controller.rb
+++ b/apps/workbench/app/controllers/actions_controller.rb
@@ -1,20 +1,77 @@
class ActionsController < ApplicationController
def combine_selected_files_into_collection
lst = []
+ files = []
params["selection"].each do |s|
m = CollectionsHelper.match(s)
- if m
+ if m[1] and m[2]
lst.append(m[1] + m[2])
+ files.append(m)
end
end
collections = Collection.where(uuid: lst)
-
- collections.each do |c|
- puts c.manifest_text
+
+ chash = {}
+ collections.each do |c|
+ c.reload()
+ chash[c.uuid] = c
+ end
+
+ combined = ""
+ files.each do |m|
+ mt = chash[m[1]+m[2]].manifest_text
+ if m[4]
+ IO.popen(['arv-normalize', '--extract', m[4][1..-1]], 'w+b') do |io|
+ io.write mt
+ io.close_write
+ while buf = io.read(2**20)
+ combined += buf
+ end
+ end
+ else
+ combined += chash[m[1]+m[2]].manifest_text
+ end
end
- '/'
+ normalized = ''
+ IO.popen(['arv-normalize'], 'w+b') do |io|
+ io.write combined
+ io.close_write
+ while buf = io.read(2**20)
+ normalized += buf
+ end
+ end
+
+ require 'digest/md5'
+
+ d = Digest::MD5.new()
+ d << normalized
+ newuuid = "#{d.hexdigest}+#{normalized.length}"
+
+ env = Hash[ENV].
+ merge({
+ 'ARVADOS_API_HOST' =>
+ $arvados_api_client.arvados_v1_base.
+ sub(/\/arvados\/v1/, '').
+ sub(/^https?:\/\//, ''),
+ 'ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token],
+ 'ARVADOS_API_HOST_INSECURE' =>
+ Rails.configuration.arvados_insecure_https ? 'true' : 'false'
+ })
+
+ IO.popen([env, 'arv-put', '--raw'], 'w+b') do |io|
+ io.write normalized
+ io.close_write
+ while buf = io.read(2**20)
+
+ end
+ end
+
+ newc = Collection.new({:uuid => newuuid, :manifest_text => normalized})
+ newc.save!
+
+ '/collections/' + newc.uuid
end
def post
diff --git a/apps/workbench/app/models/collection.rb b/apps/workbench/app/models/collection.rb
index e22a5f7..fd6b4e4 100644
--- a/apps/workbench/app/models/collection.rb
+++ b/apps/workbench/app/models/collection.rb
@@ -1,4 +1,5 @@
class Collection < ArvadosBase
+
def total_bytes
if files
tot = 0
diff --git a/apps/workbench/app/views/collections/_show_files.html.erb b/apps/workbench/app/views/collections/_show_files.html.erb
index a8535ab..6600a40 100644
--- a/apps/workbench/app/views/collections/_show_files.html.erb
+++ b/apps/workbench/app/views/collections/_show_files.html.erb
@@ -19,8 +19,7 @@
<% file_path = "#{file[0]}/#{file[1]}" %>
<tr>
<td>
- <% puts "#{file[0]} #{file[1]} #{file_path}" %>
- <%= check_box_tag 'uuids[]', @object.uuid+file_path, false, {
+ <%= check_box_tag 'uuids[]', @object.uuid+'/'+file_path, false, {
:class => 'persistent-selection',
:friendly_type => "File",
:friendly_name => "#{@object.uuid}/#{file_path}",
diff --git a/sdk/python/bin/arv-normalize b/sdk/python/bin/arv-normalize
index b1a6ca7..755b565 100755
--- a/sdk/python/bin/arv-normalize
+++ b/sdk/python/bin/arv-normalize
@@ -13,6 +13,8 @@ logger = logging.getLogger(os.path.basename(sys.argv[0]))
parser = argparse.ArgumentParser(
description='Read manifest on standard input and put normalized manifest on standard output.')
+parser.add_argument('--extract', type=str, help="The file to extract from the input manifest")
+
args = parser.parse_args()
import arvados
@@ -21,4 +23,17 @@ r = sys.stdin.read()
cr = arvados.CollectionReader(r)
-print cr.manifest_text()
+if args.extract:
+ i = args.extract.rfind('/')
+ if i == -1:
+ stream = '.'
+ fn = args.extract
+ else:
+ stream = args.extract[:i]
+ fn = args.extract[(i+1):]
+ for s in cr.all_streams():
+ if s.name() == stream:
+ if fn in s.files():
+ sys.stdout.write(s.files()[fn].as_manifest())
+else:
+ sys.stdout.write(cr.manifest_text())
commit 550e20e00518ce51054e9c52549e77f74d033eba
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 28 13:47:58 2014 -0500
POST works to actions controller, still need to implement the actual action to build the manifest.
diff --git a/apps/workbench/app/assets/javascripts/selection.js b/apps/workbench/app/assets/javascripts/selection.js
index c8ec810..ef1c592 100644
--- a/apps/workbench/app/assets/javascripts/selection.js
+++ b/apps/workbench/app/assets/javascripts/selection.js
@@ -40,7 +40,6 @@ jQuery(function($){
};
var remove_selection_click = function(e) {
- //remove_selection($(this).attr('name'));
remove_selection($(this).val());
};
@@ -49,23 +48,26 @@ jQuery(function($){
update_count();
}
+ var combine_selected_files_into_collection = function () {
+
+ }
+
var update_count = function(e) {
var lst = get_selection_list();
$("#persistent-selection-count").text(lst.length);
if (lst.length > 0) {
- $('#persistent-selection-list').html('<li><a href="#" class="btn pull-right" id="clear_selections_button">Clear selections</a></li>'
- +'<li class="notification"><table style="width: 100%"></table></li>');
+ $('#selection-form-content').html('<li><input type="submit" name="combine_selected_files_into_collection" id="combine_selected_files_into_collection" value="Combine selected collections and files into a new collection"/></li>'
+ + '<li><a href="#" id="clear_selections_button">Clear selections</a></li>'
+ + '<li class="notification"><table style="width: 100%"></table></li>');
for (var i = 0; i < lst.length; i++) {
- $('#persistent-selection-list > li > table').append("<tr>"
+ $('#selection-form-content > li > table').append("<tr>"
+ "<td>"
- + "<form>"
- + "<input class='remove-selection' type='checkbox' value='" + lst[i].uuid + "' checked='true'></input>"
- + "</form>"
+ + "<input class='remove-selection' name='selection[]' type='checkbox' value='" + lst[i].uuid + "' checked='true'></input>"
+ "</td>"
+ "<td>"
- + "<span style='padding-left: 1em'><a href=\"" + lst[i].href + "\">" + lst[i].name + "</a></span>"
+ + "<div style='padding-left: 1em'><a href=\"" + lst[i].href + "\">" + lst[i].name + "</a></div>"
+ "</td>"
+ "<td style=\"vertical-align: top\">"
@@ -93,6 +95,7 @@ jQuery(function($){
$('.remove-selection').on('click', remove_selection_click);
$('#clear_selections_button').on('click', clear_selections);
+ $('#combine_selected_files_into_collection').on('click', combine_selected_files_into_collection);
};
$(document).
diff --git a/apps/workbench/app/controllers/actions_controller.rb b/apps/workbench/app/controllers/actions_controller.rb
new file mode 100644
index 0000000..94bfd1e
--- /dev/null
+++ b/apps/workbench/app/controllers/actions_controller.rb
@@ -0,0 +1,27 @@
+class ActionsController < ApplicationController
+ def combine_selected_files_into_collection
+ lst = []
+ params["selection"].each do |s|
+ m = CollectionsHelper.match(s)
+ if m
+ lst.append(m[1] + m[2])
+ end
+ end
+
+ collections = Collection.where(uuid: lst)
+
+ collections.each do |c|
+ puts c.manifest_text
+ end
+
+ '/'
+ end
+
+ def post
+ if params["combine_selected_files_into_collection"]
+ redirect_to combine_selected_files_into_collection
+ else
+ redirect_to :back
+ end
+ end
+end
diff --git a/apps/workbench/app/controllers/application_controller.rb b/apps/workbench/app/controllers/application_controller.rb
index 412f86c..2790f96 100644
--- a/apps/workbench/app/controllers/application_controller.rb
+++ b/apps/workbench/app/controllers/application_controller.rb
@@ -4,7 +4,7 @@ class ApplicationController < ActionController::Base
around_filter :thread_clear
around_filter :thread_with_mandatory_api_token, :except => [:render_exception, :render_not_found]
around_filter :thread_with_optional_api_token
- before_filter :find_object_by_uuid, :except => [:index, :render_exception, :render_not_found]
+ before_filter :find_object_by_uuid, :except => [:index, :post, :render_exception, :render_not_found]
before_filter :check_user_agreements, :except => [:render_exception, :render_not_found]
before_filter :check_user_notifications, :except => [:render_exception, :render_not_found]
theme :select_theme
diff --git a/apps/workbench/app/helpers/collections_helper.rb b/apps/workbench/app/helpers/collections_helper.rb
index df0ba22..7b548df 100644
--- a/apps/workbench/app/helpers/collections_helper.rb
+++ b/apps/workbench/app/helpers/collections_helper.rb
@@ -6,6 +6,6 @@ module CollectionsHelper
end
def self.match(uuid)
- /^([a-f0-9]{32}(\+[0-9]+)?)(\+.*)?$/.match(uuid.to_s)
+ /^([a-f0-9]{32})(\+[0-9]+)?(\+.*?)?(\/.*)?$/.match(uuid.to_s)
end
end
diff --git a/apps/workbench/app/views/collections/_show_files.html.erb b/apps/workbench/app/views/collections/_show_files.html.erb
index 705ded3..a8535ab 100644
--- a/apps/workbench/app/views/collections/_show_files.html.erb
+++ b/apps/workbench/app/views/collections/_show_files.html.erb
@@ -19,14 +19,13 @@
<% file_path = "#{file[0]}/#{file[1]}" %>
<tr>
<td>
- <% fp2 = file_path[2..-1] if file_path[0..1] == './' %>
- <% fp2 ||= file_path %>
-<%= check_box_tag 'uuids[]', @object.uuid+file_path, false, {
- :class => 'persistent-selection',
- :friendly_type => "File",
- :friendly_name => "#{@object.uuid}/#{fp2}",
- :href => "#{url_for controller: 'collections', action: 'show', id: @object.uuid }/#{file_path}"
- } %>
+ <% puts "#{file[0]} #{file[1]} #{file_path}" %>
+ <%= check_box_tag 'uuids[]', @object.uuid+file_path, false, {
+ :class => 'persistent-selection',
+ :friendly_type => "File",
+ :friendly_name => "#{@object.uuid}/#{file_path}",
+ :href => "#{url_for controller: 'collections', action: 'show', id: @object.uuid }/#{file_path}"
+ } %>
</td>
<td>
<%= file[0] %>
diff --git a/apps/workbench/app/views/layouts/application.html.erb b/apps/workbench/app/views/layouts/application.html.erb
index 68f0e10..abef471 100644
--- a/apps/workbench/app/views/layouts/application.html.erb
+++ b/apps/workbench/app/views/layouts/application.html.erb
@@ -121,7 +121,10 @@
<span class="badge" id="persistent-selection-count"></span>
<span class="caret"></span>
</a>
- <ul class="dropdown-menu" role="menu" id="persistent-selection-list">
+ <ul class="dropdown-menu" role="menu" id="persistent-selection-list">
+ <%= form_tag '/actions' do %>
+ <div id="selection-form-content"></div>
+ <% end %>
</ul>
</li>
diff --git a/apps/workbench/config/routes.rb b/apps/workbench/config/routes.rb
index 5330a91..51aa4e0 100644
--- a/apps/workbench/config/routes.rb
+++ b/apps/workbench/config/routes.rb
@@ -33,6 +33,9 @@ ArvadosWorkbench::Application.routes.draw do
match '/collections/graph' => 'collections#graph'
resources :collections
get '/collections/:uuid/*file' => 'collections#show_file', :format => false
+
+ post 'actions' => 'actions#post'
+
root :to => 'users#welcome'
# Send unroutable requests to an arbitrary controller
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list