[ARVADOS] updated: 0d01ca86e6c73cb31efed555cbc645a8afab32eb
git at public.curoverse.com
git at public.curoverse.com
Mon Nov 24 03:54:17 EST 2014
Summary of changes:
.../app/assets/javascripts/angular_shim.js | 8 +
.../app/assets/javascripts/arvados_client.js | 46 ++++-
.../app/assets/javascripts/upload_to_collection.js | 199 ++++++++++++++-------
.../app/controllers/application_controller.rb | 14 +-
.../app/controllers/collections_controller.rb | 4 +-
.../app/views/collections/_show_files.html.erb | 43 -----
.../app/views/collections/_show_upload.html.erb | 54 ++++++
.../app/views/layouts/application.html.erb | 2 +-
apps/workbench/app/views/projects/show.html.erb | 2 +-
services/keepproxy/keepproxy.go | 16 +-
10 files changed, 258 insertions(+), 130 deletions(-)
create mode 100644 apps/workbench/app/assets/javascripts/angular_shim.js
create mode 100644 apps/workbench/app/views/collections/_show_upload.html.erb
via 0d01ca86e6c73cb31efed555cbc645a8afab32eb (commit)
via cd0ecbdddd9eeac9063d204de80dece4c75c01bb (commit)
via 60743d6764aa690d97509aa5bcaa4731f36b67a9 (commit)
via cee4332f5921946203d20c0da6bbde13e3d96622 (commit)
via 7d0d7b0c3c66242c6ccf74613277092a3ff06ebc (commit)
via cfc9ed031a0a689d92907ae201a1a18385f04ea4 (commit)
via d6a28a4d11255751c0c954c0cd1d9c0e79069e6c (commit)
from d2e9fba1946ead6c059d3df3cf34139563f2db60 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 0d01ca86e6c73cb31efed555cbc645a8afab32eb
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 24 03:52:27 2014 -0500
3781: Hide Upload tab if collection is not writable.
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index e3f1b4e..eacf8b1 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -14,7 +14,9 @@ class CollectionsController < ApplicationController
RELATION_LIMIT = 5
def show_pane_list
- %w(Files Upload Provenance_graph Used_by Advanced)
+ panes = %w(Files Upload Provenance_graph Used_by Advanced)
+ panes = panes - %w(Upload) unless (@object.editable? rescue false)
+ panes
end
def set_persistent
commit cd0ecbdddd9eeac9063d204de80dece4c75c01bb
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 24 03:37:04 2014 -0500
3781: Add files as they finish. Deal with name conflicts.
diff --git a/apps/workbench/app/assets/javascripts/arvados_client.js b/apps/workbench/app/assets/javascripts/arvados_client.js
index afab4be..a1f34d2 100644
--- a/apps/workbench/app/assets/javascripts/arvados_client.js
+++ b/apps/workbench/app/assets/javascripts/arvados_client.js
@@ -38,6 +38,41 @@ function ArvadosClient(discoveryUri, apiToken) {
this.discoveryDoc = data;
});
return this.promiseDiscovery;
+ },
+ uniqueNameForManifest: function(manifest, streamName, origName) {
+ // Return an (escaped) filename starting with (unescaped)
+ // origName that won't conflict with any existing names in
+ // the manifest if saved under streamName. streamName must
+ // be exactly as given in the manifest, e.g., "." or
+ // "./foo" or "./foo/bar".
+ //
+ // Example:
+ //
+ // unique('./foo [...] 0:0:bar\040baz\n', '.', 'foo/bar baz')
+ // =>
+ // 'foo/bar\\040baz\\040(1)'
+ var newName;
+ var nameStub = origName;
+ var suffixInt = null;
+ var ok = false;
+ while (!ok) {
+ ok = true;
+ // Add ' (N)' before the filename extension, if any.
+ newName = (!suffixInt ? nameStub :
+ nameStub.replace(/(\.[^.]*)?$/, ' ('+suffixInt+')$1')).
+ replace(/ /g, '\\040');
+ $.each(manifest.split('\n'), function(_, line) {
+ var i, match, foundName;
+ var toks = line.split(' ');
+ for (var i=1; i<toks.length && ok; i++)
+ if (match = toks[i].match(/^\d+:\d+:(\S+)/))
+ if (toks[0] + '/' + match[1] === streamName + '/' + newName) {
+ suffixInt = (suffixInt || 0) + 1;
+ ok = false;
+ }
+ });
+ }
+ return newName;
}
});
}
diff --git a/apps/workbench/app/assets/javascripts/upload_to_collection.js b/apps/workbench/app/assets/javascripts/upload_to_collection.js
index dac341e..8a1d354 100644
--- a/apps/workbench/app/assets/javascripts/upload_to_collection.js
+++ b/apps/workbench/app/assets/javascripts/upload_to_collection.js
@@ -25,8 +25,12 @@ function UploadController($scope, $q, $timeout, numberFilter, dateFilter) {
// inputs, so we need to $scope.$apply() this update.
$scope.$apply(function(){
var i;
+ var insertAt;
+ for (insertAt=0; (insertAt<$scope.uploadQueue.length &&
+ $scope.uploadQueue[insertAt].state != 'Done');
+ insertAt++);
for (i=0; i<files.length; i++) {
- $scope.uploadQueue.push(
+ $scope.uploadQueue.splice(insertAt+i, 0,
new FileUploader($scope, $q, files[i]));
}
});
@@ -157,6 +161,7 @@ function SliceUploader(uploader, label, data, dataSize) {
function FileUploader($scope, $q, file) {
var that = this;
$.extend(this, {
+ committed: false,
file: file,
locators: [],
progress: 0.0,
@@ -202,7 +207,7 @@ function FileUploader($scope, $q, file) {
if (!this._currentSlice) {
this.state = 'Done';
this._currentUploader = null;
- this._deferred.resolve('Done');
+ this._deferred.resolve([this]);
return;
}
this._currentUploader = new SliceUploader(
@@ -315,32 +320,27 @@ function QueueUploader($scope, $q, $timeout) {
return that.doQueueWork();
},
doQueueWork: function() {
- var i, firstdone;
+ var i;
that.state = 'Running';
that.stateReason = null;
- if ($scope.uploadQueue.length == 0)
- return that.onQueueResolve();
- // Make the not-done items rise to the top of the queue.
- for (i=0, firstdone=0; i<$scope.uploadQueue.length; i++) {
- if ($scope.uploadQueue[i].state == 'Done') {
- firstdone = i;
- } else {
- // Move item [i] up to item [firstdone], move
- // intervening items down.
- $scope.uploadQueue.splice(firstdone, 0,
- $scope.uploadQueue.splice(i, 1)[0]);
- firstdone++;
- }
- }
+ // Push the done things to the bottom of the queue.
+ for (i=0; (i<$scope.uploadQueue.length &&
+ $scope.uploadQueue[i].state == 'Done'); i++);
+ if (i>0)
+ $scope.uploadQueue.push.apply($scope.uploadQueue, $scope.uploadQueue.splice(0, i));
// If anything is not-done, do it.
- if ($scope.uploadQueue[0].state != 'Done') {
+ if ($scope.uploadQueue.length > 0 &&
+ $scope.uploadQueue[0].state != 'Done') {
return $scope.uploadQueue[0].go().then(
- that.doQueueWork,
+ that.appendToCollection,
that.onQueueReject,
- that.onQueueProgress);
+ that.onQueueProgress
+ ).then(
+ that.doQueueWork,
+ that.onQueueReject);
}
- // If everything is done, write the manifest.
- return that.appendToCollection();
+ // If everything is done, resolve the promise and clean up.
+ return that.onQueueResolve();
},
onQueueReject: function(reason) {
that.state = 'Failed';
@@ -366,31 +366,40 @@ function QueueUploader($scope, $q, $timeout) {
// Ensure updates happen after FileUpload promise callbacks.
$timeout(function(){$scope.$apply();});
},
- appendToCollection: function() {
- var manifestText = '';
- var upload, i;
- for (i=0; i<$scope.uploadQueue.length; i++) {
- upload = $scope.uploadQueue[i];
- manifestText += '. ' +
- upload.locators.join(' ') +
- ' 0:' + upload.file.size.toString() + ':' +
- upload.file.name.replace(/ /g, '\\040') +
- '\n';
- }
+ appendToCollection: function(uploads) {
+ var deferred = $q.defer();
return $scope.uploader.arvados.apiPromise(
'collections', 'get',
{ uuid: $scope.uuid }).
then(function(collection) {
+ var manifestText = '';
+ var upload, i;
+ for (i=0; i<uploads.length; i++) {
+ upload = uploads[i];
+ filename = $scope.uploader.arvados.uniqueNameForManifest(
+ collection.manifest_text,
+ '.', upload.file.name);
+ collection.manifest_text += '. ' +
+ upload.locators.join(' ') +
+ ' 0:' + upload.file.size.toString() + ':' +
+ filename +
+ '\n';
+ }
return $scope.uploader.arvados.apiPromise(
'collections', 'update',
{ uuid: $scope.uuid,
collection:
{ manifest_text:
- '' + collection.manifest_text + manifestText }
+ collection.manifest_text }
}).
- then(that.onQueueResolve);
- }, that.onUploaderReject);
- // The page will automatically refresh because websockets.
+ then(deferred.resolve);
+ }, that.onUploaderReject).then(function() {
+ var i;
+ for(i=0; i<uploads.length; i++) {
+ uploads[i].committed = true;
+ }
+ });
+ return deferred;
}
});
}
diff --git a/apps/workbench/app/views/collections/_show_upload.html.erb b/apps/workbench/app/views/collections/_show_upload.html.erb
index 40c2f32..c20cea3 100644
--- a/apps/workbench/app/views/collections/_show_upload.html.erb
+++ b/apps/workbench/app/views/collections/_show_upload.html.erb
@@ -1,49 +1,54 @@
-<div class="panel panel-info">
- <div class="panel-heading"><h3 class="panel-title">Upload files</h3></div>
- <div class="panel-body" ng-controller="UploadController" uuid="<%= @object.uuid %>">
- <div class="panel panel-primary">
- <div class="panel-body">
- <div class="row">
- <div class="col-sm-4">
- <input type="file" multiple ng-model="incoming" onchange="angular.element(this).scope().addFilesToQueue(this.files); $(this).val('');">
- <div class="btn-group btn-group-sm" role="group" style="margin-top: .5em">
- <button type="button" class="btn btn-default" ng-click="stop()" ng-disabled="uploader.state != 'Running'"><i class="fa fa-fw fa-pause"></i> Pause</button>
- <button type="button" class="btn btn-primary" ng-click="go()" ng-disabled="uploader.state == 'Running' || uploadQueue.length == 0"><i class="fa fa-fw fa-play"></i> Start</button>
- </div>
+<div class="arv-log-refresh-control"
+ data-load-throttle="86486400000" <%# 1001 nights %>
+ ></div>
+<div ng-controller="UploadController" uuid="<%= @object.uuid %>">
+ <div class="panel panel-primary">
+ <div class="panel-body">
+ <div class="row">
+ <div class="col-sm-4">
+ <input type="file" multiple ng-model="incoming" onchange="angular.element(this).scope().addFilesToQueue(this.files); $(this).val('');">
+ <div class="btn-group btn-group-sm" role="group" style="margin-top: .5em">
+ <button type="button" class="btn btn-default" ng-click="stop()" ng-disabled="uploader.state != 'Running'"><i class="fa fa-fw fa-pause"></i> Pause</button>
+ <button type="button" class="btn btn-primary" ng-click="go()" ng-disabled="uploader.state == 'Running' || uploadQueue.length == 0"><i class="fa fa-fw fa-play"></i> Start</button>
</div>
- <div class="col-sm-8">
- <div ng-show="uploader.state == 'Idle' && uploader.stateReason"
- class="alert alert-success"
- ><i class="fa fa-flag-checkered"></i> {{uploader.stateReason}}
- </div>
- <div ng-show="uploader.state == 'Failed'"
- class="alert alert-danger"
- ><i class="fa fa-warning"></i> {{uploader.stateReason}}
- </div>
+ </div>
+ <div class="col-sm-8">
+ <div ng-show="uploader.state == 'Idle' && uploader.stateReason"
+ class="alert alert-success"
+ ><i class="fa fa-flag-checkered"></i> {{uploader.stateReason}}
+ </div>
+ <div ng-show="uploader.state == 'Failed'"
+ class="alert alert-danger"
+ ><i class="fa fa-warning"></i> {{uploader.stateReason}}
</div>
</div>
</div>
</div>
- <div ng-repeat="upload in uploadQueue" class="row">
- <div class="col-sm-1">
- <button class="btn btn-xs btn-default" ng-click="removeFileFromQueue($index)" title="cancel"><i class="fa fa-fw fa-trash-o"></i></button>
- </div>
- <div class="col-sm-4 nowrap" style="overflow-x:hidden;text-overflow:ellipsis">
- <span title="{{upload.file.name}}">
- {{upload.file.name}}
- </span>
- </div>
- <div class="col-sm-1" style="text-align: right">
- {{upload.file.size/1024 | number:0}}K
- </div>
- <div class="col-sm-2">
- <div class="progress">
- <span class="progress-bar" style="width: {{upload.progress}}%"></span>
- </div>
- </div>
- <div class="col-sm-4" ng-class="{lighten: upload.state != 'Uploading'}">
- {{upload.statistics}}
+ </div>
+ <div ng-repeat="upload in uploadQueue" class="row" ng-class="{lighten: upload.committed}">
+ <div class="col-sm-1">
+ <button class="btn btn-xs btn-default"
+ ng-show="!upload.committed"
+ ng-click="removeFileFromQueue($index)"
+ title="cancel"><i class="fa fa-fw fa-trash-o"></i></button>
+ <span class="label label-success label-info"
+ ng-show="upload.committed">added</span>
+ </div>
+ <div class="col-sm-4 nowrap" style="overflow-x:hidden;text-overflow:ellipsis">
+ <span title="{{upload.file.name}}">
+ {{upload.file.name}}
+ </span>
+ </div>
+ <div class="col-sm-1" style="text-align: right">
+ {{upload.file.size/1024 | number:0}}K
+ </div>
+ <div class="col-sm-2">
+ <div class="progress">
+ <span class="progress-bar" style="width: {{upload.progress}}%"></span>
</div>
</div>
+ <div class="col-sm-4" ng-class="{lighten: upload.state != 'Uploading'}">
+ {{upload.statistics}}
+ </div>
</div>
</div>
commit 60743d6764aa690d97509aa5bcaa4731f36b67a9
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 24 03:36:07 2014 -0500
3781: Start at the Upload tab when creating a collection via "add data".
diff --git a/apps/workbench/app/controllers/application_controller.rb b/apps/workbench/app/controllers/application_controller.rb
index 3270cfb..6fea625 100644
--- a/apps/workbench/app/controllers/application_controller.rb
+++ b/apps/workbench/app/controllers/application_controller.rb
@@ -256,7 +256,9 @@ class ApplicationController < ActionController::Base
elsif request.method.in? ['GET', 'HEAD']
render
else
- redirect_to params[:return_to] || @object
+ redirect_to (params[:return_to] ||
+ polymorphic_url(@object,
+ anchor: params[:redirect_to_anchor]))
end
}
f.js { render }
@@ -321,15 +323,9 @@ class ApplicationController < ActionController::Base
@new_resource_attrs.reject! { |k,v| k.to_s == 'uuid' }
@object ||= model_class.new @new_resource_attrs, params["options"]
if @object.save
- respond_to do |f|
- f.json { render json: @object.attributes.merge(href: url_for(action: :show, id: @object)) }
- f.html {
- redirect_to @object
- }
- f.js { render }
- end
+ show
else
- self.render_error status: 422
+ render_error status: 422
end
end
diff --git a/apps/workbench/app/views/projects/show.html.erb b/apps/workbench/app/views/projects/show.html.erb
index 228279a..9869577 100644
--- a/apps/workbench/app/views/projects/show.html.erb
+++ b/apps/workbench/app/views/projects/show.html.erb
@@ -23,7 +23,7 @@
<% end %>
</li>
<li>
- <%= link_to(collections_path(collection: {manifest_text: "", owner_uuid: @object.uuid}),
+ <%= link_to(collections_path(collection: {manifest_text: "", owner_uuid: @object.uuid}, redirect_to_anchor: 'Upload'),
{ method: 'post', title: "Add data to this project", data: {'event-after-select' => 'page-refresh', 'toggle' => 'dropdown'} }) do %>
<i class="fa fa-fw fa-upload"></i> ...from your computer
<% end %>
commit cee4332f5921946203d20c0da6bbde13e3d96622
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 24 02:07:56 2014 -0500
3781: Move uploader to its own tab.
diff --git a/apps/workbench/app/assets/javascripts/angular_shim.js b/apps/workbench/app/assets/javascripts/angular_shim.js
new file mode 100644
index 0000000..f329cb7
--- /dev/null
+++ b/apps/workbench/app/assets/javascripts/angular_shim.js
@@ -0,0 +1,8 @@
+// Compile any new HTML content that was loaded via ajax.
+
+$(document).on('ajax:success arv:pane:loaded', function() {
+ angular.element(document).injector().invoke(function($compile) {
+ var scope = angular.element(document).scope();
+ $compile(document)(scope);
+ });
+});
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 39f637e..e3f1b4e 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -14,7 +14,7 @@ class CollectionsController < ApplicationController
RELATION_LIMIT = 5
def show_pane_list
- %w(Files Provenance_graph Used_by Advanced)
+ %w(Files Upload Provenance_graph Used_by Advanced)
end
def set_persistent
diff --git a/apps/workbench/app/views/collections/_show_files.html.erb b/apps/workbench/app/views/collections/_show_files.html.erb
index 6420bd0..76d8731 100644
--- a/apps/workbench/app/views/collections/_show_files.html.erb
+++ b/apps/workbench/app/views/collections/_show_files.html.erb
@@ -63,55 +63,6 @@ function unselect_all_files() {
<% file_tree = @object.andand.files_tree %>
<% if file_tree.nil? or file_tree.empty? %>
<p>This collection is empty.</p>
- <div ng-app="Workbench" class="panel panel-info">
- <div class="panel-heading"><h3 class="panel-title">Upload files</h3></div>
- <div class="panel-body" ng-controller="UploadController" uuid="<%= @object.uuid %>">
- <div class="panel panel-primary">
- <div class="panel-body">
- <div class="row">
- <div class="col-sm-4">
- <input type="file" multiple ng-model="incoming" onchange="angular.element(this).scope().addFilesToQueue(this.files); $(this).val('');">
- <div class="btn-group btn-group-sm" role="group" style="margin-top: .5em">
- <button type="button" class="btn btn-default" ng-click="stop()" ng-disabled="uploader.state != 'Running'"><i class="fa fa-fw fa-pause"></i> Pause</button>
- <button type="button" class="btn btn-primary" ng-click="go()" ng-disabled="uploader.state == 'Running' || uploadQueue.length == 0"><i class="fa fa-fw fa-play"></i> Start</button>
- </div>
- </div>
- <div class="col-sm-8">
- <div ng-show="uploader.state == 'Idle' && uploader.stateReason"
- class="alert alert-success"
- ><i class="fa fa-flag-checkered"></i> {{uploader.stateReason}}
- </div>
- <div ng-show="uploader.state == 'Failed'"
- class="alert alert-danger"
- ><i class="fa fa-warning"></i> {{uploader.stateReason}}
- </div>
- </div>
- </div>
- </div>
- </div>
- <div ng-repeat="upload in uploadQueue" class="row">
- <div class="col-sm-1">
- <button class="btn btn-xs btn-default" ng-click="removeFileFromQueue($index)" title="cancel"><i class="fa fa-fw fa-trash-o"></i></button>
- </div>
- <div class="col-sm-4 nowrap" style="overflow-x:hidden;text-overflow:ellipsis">
- <span title="{{upload.file.name}}">
- {{upload.file.name}}
- </span>
- </div>
- <div class="col-sm-1" style="text-align: right">
- {{upload.file.size/1024 | number:0}}K
- </div>
- <div class="col-sm-2">
- <div class="progress">
- <span class="progress-bar" style="width: {{upload.progress}}%"></span>
- </div>
- </div>
- <div class="col-sm-4" ng-class="{lighten: upload.state != 'Uploading'}">
- {{upload.statistics}}
- </div>
- </div>
- </div>
- </div>
<% else %>
<ul id="collection_files" class="collection_files <%=preview_selectable_container%>">
<% dirstack = [file_tree.first.first] %>
diff --git a/apps/workbench/app/views/collections/_show_upload.html.erb b/apps/workbench/app/views/collections/_show_upload.html.erb
new file mode 100644
index 0000000..40c2f32
--- /dev/null
+++ b/apps/workbench/app/views/collections/_show_upload.html.erb
@@ -0,0 +1,49 @@
+<div class="panel panel-info">
+ <div class="panel-heading"><h3 class="panel-title">Upload files</h3></div>
+ <div class="panel-body" ng-controller="UploadController" uuid="<%= @object.uuid %>">
+ <div class="panel panel-primary">
+ <div class="panel-body">
+ <div class="row">
+ <div class="col-sm-4">
+ <input type="file" multiple ng-model="incoming" onchange="angular.element(this).scope().addFilesToQueue(this.files); $(this).val('');">
+ <div class="btn-group btn-group-sm" role="group" style="margin-top: .5em">
+ <button type="button" class="btn btn-default" ng-click="stop()" ng-disabled="uploader.state != 'Running'"><i class="fa fa-fw fa-pause"></i> Pause</button>
+ <button type="button" class="btn btn-primary" ng-click="go()" ng-disabled="uploader.state == 'Running' || uploadQueue.length == 0"><i class="fa fa-fw fa-play"></i> Start</button>
+ </div>
+ </div>
+ <div class="col-sm-8">
+ <div ng-show="uploader.state == 'Idle' && uploader.stateReason"
+ class="alert alert-success"
+ ><i class="fa fa-flag-checkered"></i> {{uploader.stateReason}}
+ </div>
+ <div ng-show="uploader.state == 'Failed'"
+ class="alert alert-danger"
+ ><i class="fa fa-warning"></i> {{uploader.stateReason}}
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div ng-repeat="upload in uploadQueue" class="row">
+ <div class="col-sm-1">
+ <button class="btn btn-xs btn-default" ng-click="removeFileFromQueue($index)" title="cancel"><i class="fa fa-fw fa-trash-o"></i></button>
+ </div>
+ <div class="col-sm-4 nowrap" style="overflow-x:hidden;text-overflow:ellipsis">
+ <span title="{{upload.file.name}}">
+ {{upload.file.name}}
+ </span>
+ </div>
+ <div class="col-sm-1" style="text-align: right">
+ {{upload.file.size/1024 | number:0}}K
+ </div>
+ <div class="col-sm-2">
+ <div class="progress">
+ <span class="progress-bar" style="width: {{upload.progress}}%"></span>
+ </div>
+ </div>
+ <div class="col-sm-4" ng-class="{lighten: upload.state != 'Uploading'}">
+ {{upload.statistics}}
+ </div>
+ </div>
+ </div>
+</div>
diff --git a/apps/workbench/app/views/layouts/application.html.erb b/apps/workbench/app/views/layouts/application.html.erb
index 6fab360..91c3c55 100644
--- a/apps/workbench/app/views/layouts/application.html.erb
+++ b/apps/workbench/app/views/layouts/application.html.erb
@@ -1,5 +1,5 @@
<!DOCTYPE html>
-<html>
+<html ng-app="Workbench">
<head>
<meta charset="utf-8">
<title>
commit 7d0d7b0c3c66242c6ccf74613277092a3ff06ebc
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 24 00:34:24 2014 -0500
3781: Fix error handling.
diff --git a/apps/workbench/app/assets/javascripts/upload_to_collection.js b/apps/workbench/app/assets/javascripts/upload_to_collection.js
index 532760d..dac341e 100644
--- a/apps/workbench/app/assets/javascripts/upload_to_collection.js
+++ b/apps/workbench/app/assets/javascripts/upload_to_collection.js
@@ -90,6 +90,23 @@ function SliceUploader(uploader, label, data, dataSize) {
// fail. Return a promise that will get resolved with
// resolve(locator) when the block is accepted by the
// proxy.
+ this._deferred = $.Deferred();
+ this.goSend();
+ return this._deferred.promise();
+ },
+ stop: function() {
+ this._failMax = 0;
+ this._jqxhr.abort();
+ },
+ _data: data,
+ _dataSize: dataSize,
+ _deferred: null,
+ _failCount: 0,
+ _failMax: 3,
+ _label: label,
+ _jqxhr: null,
+ _uploader: uploader,
+ goSend: function() {
this._jqxhr = $.ajax({
url: this.proxyUriBase(),
type: 'POST',
@@ -102,44 +119,31 @@ function SliceUploader(uploader, label, data, dataSize) {
xhr: function() {
var xhr = $.ajaxSettings.xhr();
if (xhr.upload) {
- xhr.upload.addEventListener(
- 'progress', this.progressSendSlice);
+ xhr.upload.onprogress = that.onSendProgress;
}
return xhr;
},
processData: false,
data: that._data
});
- return this._jqxhr.then(this.doneSendSlice, this.failSendSlice);
- },
- stop: function() {
- this._failMax = 0;
- this._jqxhr.abort();
+ this._jqxhr.then(this.onSendResolve, this.onSendReject);
},
- _data: data,
- _dataSize: dataSize,
- _failCount: 0,
- _failMax: 3,
- _label: label,
- _jqxhr: null,
- _uploader: uploader,
- progressSendSlice: function(x,y,z) {
- console.log(['uploadProgress',x,y,z]);
- that._jqxhr.notify(50, that._dataSize);
+ onSendProgress: function(xhrProgressEvent) {
+ that._deferred.notify(xhrProgressEvent.position, that._dataSize);
},
- doneSendSlice: function(data, textStatus, jqxhr) {
- return $.Deferred().resolve(data, that._dataSize).promise();
+ onSendResolve: function(data, textStatus, jqxhr) {
+ that._deferred.resolve(data, that._dataSize);
},
- failSendSlice: function(xhr, textStatus, err) {
- if (++that._failCount <= that._failMax) {
+ onSendReject: function(xhr, textStatus, err) {
+ if (++that._failCount < that._failMax) {
// TODO: nice to tell the user that retry is happening.
console.log('slice ' + that._label + ': ' +
textStatus + ', retry ' + that._failCount);
- return that.go();
+ that.goSend();
+ } else {
+ that._deferred.reject(
+ {xhr: xhr, textStatus: textStatus, err: err});
}
- // Can't propagate multiple arguments with "return a,b,c"
- // so we do this indirectly by returning a new promise.
- return $.Deferred().reject(xhr, textStatus, err).promise();
},
proxyUriBase: function() {
var proxy = this._uploader.keepProxy;
@@ -162,6 +166,7 @@ function FileUploader($scope, $q, file) {
if (this._deferred)
this._deferred.reject(null, 'Restarted', null);
this._deferred = $q.defer();
+ this.statistics = 'Starting';
this.state = 'Uploading';
this.startTime = Date.now();
this.startByte = this._readPos;
@@ -212,18 +217,24 @@ function FileUploader($scope, $q, file) {
},
onUploaderResolve: function(locator, dataSize) {
// TODO: check that._currentSlice.size == dataSize
+ if (!locator || that._currentSlice.size != dataSize) {
+ console.log("onUploaderResolve but locator=" + locator +
+ " and " + that._currentSlice.size + " != " + dataSize);
+ return that.onUploaderReject(
+ null, "Error", "Bad response from slice upload");
+ }
that.locators.push(locator);
that._readPos += dataSize;
that.goSlice();
},
- onUploaderReject: function(xhr, status, err) {
+ onUploaderReject: function(reason) {
+ that.statistics = 'Interrupted';
that.state = 'Paused';
that.setProgress(that._readPos);
- that._deferred.reject(xhr, status, err);
+ that._deferred.reject(reason);
},
onUploaderProgress: function(sliceDone, sliceSize) {
- that.setProgress(that._readPos - sliceSize + sliceDone);
- console.log("upload progress: " + that.progress);
+ that.setProgress(that._readPos + sliceDone);
},
nextSlice: function() {
var size = Math.min(
@@ -241,9 +252,7 @@ function FileUploader($scope, $q, file) {
setProgress: function(bytesDone) {
var kBps;
this.progress = Math.min(100, 100 * bytesDone / this.file.size)
- if (bytesDone <= this.startByte) {
- this.statistics = 'Starting';
- } else {
+ if (bytesDone > this.startByte) {
kBps = (bytesDone - this.startByte) /
(Date.now() - this.startTime);
this.statistics = (
@@ -251,7 +260,7 @@ function FileUploader($scope, $q, file) {
'at ~' + this.$scope.numberFilter(kBps, '0') + 'K/s')
if (this.state == 'Paused') {
// That's all I have to say about that.
- } else if (bytesDone < this.file.size) {
+ } else if (that.state == 'Uploading') {
this.statistics += ', ETA ' +
this.$scope.dateFilter(
new Date(
@@ -263,7 +272,6 @@ function FileUploader($scope, $q, file) {
this.finishTime = Date.now();
}
}
- console.log(this.progress);
this._deferred.notify();
}
});
@@ -283,7 +291,7 @@ function QueueUploader($scope, $q, $timeout) {
'keep_services', 'list',
{filters: [['service_type','=','proxy']]}).
then(this.doQueueWithProxy);
- this.reportUploadProgress();
+ this.onQueueProgress();
return this._deferred.promise();
},
stop: function() {
@@ -301,50 +309,68 @@ function QueueUploader($scope, $q, $timeout) {
that.state = 'Failed';
that.stateReason =
'There seems to be no Keep proxy service available.';
- that._deferred.reject();
+ that._deferred.reject(null, 'error', that.stateReason);
return;
}
return that.doQueueWork();
},
doQueueWork: function() {
+ var i, firstdone;
that.state = 'Running';
that.stateReason = null;
- for (var i=0; i<$scope.uploadQueue.length; i++) {
- if ($scope.uploadQueue[i].state != 'Done') {
- return $scope.uploadQueue[i].go().then(
- that.doQueueWork,
- that.reportUploadError,
- that.reportUploadProgress);
+ if ($scope.uploadQueue.length == 0)
+ return that.onQueueResolve();
+ // Make the not-done items rise to the top of the queue.
+ for (i=0, firstdone=0; i<$scope.uploadQueue.length; i++) {
+ if ($scope.uploadQueue[i].state == 'Done') {
+ firstdone = i;
+ } else {
+ // Move item [i] up to item [firstdone], move
+ // intervening items down.
+ $scope.uploadQueue.splice(firstdone, 0,
+ $scope.uploadQueue.splice(i, 1)[0]);
+ firstdone++;
}
}
+ // If anything is not-done, do it.
+ if ($scope.uploadQueue[0].state != 'Done') {
+ return $scope.uploadQueue[0].go().then(
+ that.doQueueWork,
+ that.onQueueReject,
+ that.onQueueProgress);
+ }
+ // If everything is done, write the manifest.
return that.appendToCollection();
},
- reportUploadError: function(jqxhr, textStatus, err) {
+ onQueueReject: function(reason) {
that.state = 'Failed';
that.stateReason = (
- (textStatus || 'Error') +
- (jqxhr && jqxhr.options
- ? (' (from ' + jqxhr.options.url + ')')
+ (reason.textStatus || 'Error') +
+ (reason.xhr && reason.xhr.options
+ ? (' (from ' + reason.xhr.options.url + ')')
: '') +
': ' +
- (err || '(no further details available, sorry!)'));
- that._deferred.reject();
- that.reportUploadProgress();
+ (reason.err || ''));
+ if (reason.xhr && reason.xhr.responseText)
+ that.stateReason += ' -- ' + reason.xhr.responseText;
+ that._deferred.reject(reason);
+ that.onQueueProgress();
},
- reportUploadSuccess: function(message) {
+ onQueueResolve: function() {
that.state = 'Idle';
- that.stateReason = message;
+ that.stateReason = 'Done!';
that._deferred.resolve();
- that.reportUploadProgress();
+ that.onQueueProgress();
},
- reportUploadProgress: function() {
+ onQueueProgress: function() {
// Ensure updates happen after FileUpload promise callbacks.
$timeout(function(){$scope.$apply();});
},
appendToCollection: function() {
var manifestText = '';
- for (var i=0; i<$scope.uploadQueue.length; i++) {
- var upload = $scope.uploadQueue[i];
+ var upload, i;
+ for (i=0; i<$scope.uploadQueue.length; i++) {
+ upload = $scope.uploadQueue[i];
manifestText += '. ' +
upload.locators.join(' ') +
' 0:' + upload.file.size.toString() + ':' +
@@ -362,11 +388,9 @@ function QueueUploader($scope, $q, $timeout) {
{ manifest_text:
'' + collection.manifest_text + manifestText }
}).
- then(that.onAppendResolve);
+ then(that.onQueueResolve);
}, that.onUploaderReject);
- },
- onAppendResolve: function() {
- that.reportUploadSuccess("Done!");
+ // The page will automatically refresh because websockets.
}
});
}
diff --git a/apps/workbench/app/views/collections/_show_files.html.erb b/apps/workbench/app/views/collections/_show_files.html.erb
index 7bd7eec..6420bd0 100644
--- a/apps/workbench/app/views/collections/_show_files.html.erb
+++ b/apps/workbench/app/views/collections/_show_files.html.erb
@@ -69,23 +69,29 @@ function unselect_all_files() {
<div class="panel panel-primary">
<div class="panel-body">
<div class="row">
- <div class="col-sm-6">
- <input type="file" multiple ng-model="incoming" onchange="angular.element(this).scope().addFilesToQueue(this.files); $(this).val('');" />
- </div>
- <div class="col-sm-6">
- <div class="btn-group btn-group-sm pull-right" role="group">
+ <div class="col-sm-4">
+ <input type="file" multiple ng-model="incoming" onchange="angular.element(this).scope().addFilesToQueue(this.files); $(this).val('');">
+ <div class="btn-group btn-group-sm" role="group" style="margin-top: .5em">
<button type="button" class="btn btn-default" ng-click="stop()" ng-disabled="uploader.state != 'Running'"><i class="fa fa-fw fa-pause"></i> Pause</button>
<button type="button" class="btn btn-primary" ng-click="go()" ng-disabled="uploader.state == 'Running' || uploadQueue.length == 0"><i class="fa fa-fw fa-play"></i> Start</button>
</div>
</div>
+ <div class="col-sm-8">
+ <div ng-show="uploader.state == 'Idle' && uploader.stateReason"
+ class="alert alert-success"
+ ><i class="fa fa-flag-checkered"></i> {{uploader.stateReason}}
+ </div>
+ <div ng-show="uploader.state == 'Failed'"
+ class="alert alert-danger"
+ ><i class="fa fa-warning"></i> {{uploader.stateReason}}
+ </div>
+ </div>
</div>
</div>
</div>
- <div ng-show="uploader.statusSuccess" class="alert alert-success"><i class="fa fa-flag-checkered"></i> {{uploader.statusSuccess}}</div>
- <div ng-show="uploader.statusError" class="alert alert-danger"><i class="fa fa-warning"></i> {{uploader.statusError}}</div>
<div ng-repeat="upload in uploadQueue" class="row">
<div class="col-sm-1">
- <button class="btn btn-xs btn-default" ng-click="removeFileFromQueue($index)" title="cancel" ng-show="upload.state != 'Done'"><i class="fa fa-fw fa-trash-o"></i></button>
+ <button class="btn btn-xs btn-default" ng-click="removeFileFromQueue($index)" title="cancel"><i class="fa fa-fw fa-trash-o"></i></button>
</div>
<div class="col-sm-4 nowrap" style="overflow-x:hidden;text-overflow:ellipsis">
<span title="{{upload.file.name}}">
diff --git a/apps/workbench/app/views/projects/show.html.erb b/apps/workbench/app/views/projects/show.html.erb
index 7769e1b..228279a 100644
--- a/apps/workbench/app/views/projects/show.html.erb
+++ b/apps/workbench/app/views/projects/show.html.erb
@@ -23,7 +23,7 @@
<% end %>
</li>
<li>
- <%= link_to(collections_path(collection: {manifest_text: ""}),
+ <%= link_to(collections_path(collection: {manifest_text: "", owner_uuid: @object.uuid}),
{ method: 'post', title: "Add data to this project", data: {'event-after-select' => 'page-refresh', 'toggle' => 'dropdown'} }) do %>
<i class="fa fa-fw fa-upload"></i> ...from your computer
<% end %>
commit cfc9ed031a0a689d92907ae201a1a18385f04ea4
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 23 23:28:34 2014 -0500
3781: Fix error propagation.
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index e547efd..503642d 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -391,7 +391,7 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
// Now try to put the block through
var replicas int
- var err error
+ var put_err error
if hash == "" {
if bytes, err := ioutil.ReadAll(req.Body); err != nil {
msg := fmt.Sprintf("Error reading request body: %s", err)
@@ -399,16 +399,16 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
http.Error(resp, msg, http.StatusInternalServerError)
return
} else {
- hash, replicas, err = kc.PutB(bytes)
+ hash, replicas, put_err = kc.PutB(bytes)
}
} else {
- hash, replicas, err = kc.PutHR(hash, req.Body, contentLength)
+ hash, replicas, put_err = kc.PutHR(hash, req.Body, contentLength)
}
// Tell the client how many successful PUTs we accomplished
resp.Header().Set(keepclient.X_Keep_Replicas_Stored, fmt.Sprintf("%d", replicas))
- switch err {
+ switch put_err {
case nil:
// Default will return http.StatusOK
log.Printf("%s: %s %s finished, stored %v replicas (desired %v)", GetRemoteAddress(req), req.Method, hash, replicas, kc.Want_replicas)
@@ -432,15 +432,15 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
log.Printf("%s: wrote %v bytes to response body and got error %v", n, err2.Error())
}
} else {
- http.Error(resp, "", http.StatusServiceUnavailable)
+ http.Error(resp, put_err.Error(), http.StatusServiceUnavailable)
}
default:
- http.Error(resp, err.Error(), http.StatusBadGateway)
+ http.Error(resp, put_err.Error(), http.StatusBadGateway)
}
- if err != nil {
- log.Printf("%s: %s %s stored %v replicas (desired %v) got error %v", GetRemoteAddress(req), req.Method, hash, replicas, kc.Want_replicas, err.Error())
+ if put_err != nil {
+ log.Printf("%s: %s %s stored %v replicas (desired %v) got error %v", GetRemoteAddress(req), req.Method, hash, replicas, kc.Want_replicas, put_err.Error())
}
}
commit d6a28a4d11255751c0c954c0cd1d9c0e79069e6c
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 23 22:27:18 2014 -0500
3781: Save the collection after all files are uploaded.
diff --git a/apps/workbench/app/assets/javascripts/arvados_client.js b/apps/workbench/app/assets/javascripts/arvados_client.js
index 92db84f..afab4be 100644
--- a/apps/workbench/app/assets/javascripts/arvados_client.js
+++ b/apps/workbench/app/assets/javascripts/arvados_client.js
@@ -4,15 +4,20 @@ function ArvadosClient(discoveryUri, apiToken) {
discoveryUri: discoveryUri,
apiPromise: function(controller, action, params) {
return this.getDiscoveryDoc().then(function() {
- meth = this.discoveryDoc.resources[controller].methods[action];
- data = $.extend({}, params, {_method: meth.httpMethod});
+ var meth = this.discoveryDoc.resources[controller].methods[action];
+ var data = $.extend({}, params, {_method: meth.httpMethod});
$.each(data, function(k, v) {
if (typeof(v) == 'object') {
data[k] = JSON.stringify(v);
}
});
+ var path = meth.path.replace(/{(.*?)}/, function(_, key) {
+ var val = data[key];
+ delete data[key];
+ return encodeURIComponent(val);
+ });
return $.ajax({
- url: this.discoveryDoc.baseUrl + meth.path,
+ url: this.discoveryDoc.baseUrl + path,
type: 'POST',
crossDomain: true,
dataType: 'json',
diff --git a/apps/workbench/app/assets/javascripts/upload_to_collection.js b/apps/workbench/app/assets/javascripts/upload_to_collection.js
index bbe76eb..532760d 100644
--- a/apps/workbench/app/assets/javascripts/upload_to_collection.js
+++ b/apps/workbench/app/assets/javascripts/upload_to_collection.js
@@ -1,7 +1,18 @@
var app = angular.module('Workbench', []);
-app.controller(
- 'UploadController',
- ['$scope', '$q', '$timeout', 'numberFilter', 'dateFilter', UploadController]);
+app.
+ directive('uuid', function() {
+ // Copy the given uuid into the current $scope.
+ return {
+ restrict: 'A',
+ link: function(scope, element, attributes) {
+ scope.uuid = attributes.uuid;
+ }
+ }
+ }).
+ controller(
+ 'UploadController',
+ ['$scope', '$q', '$timeout', 'numberFilter', 'dateFilter',
+ UploadController]);
function UploadController($scope, $q, $timeout, numberFilter, dateFilter) {
$.extend($scope, {
@@ -142,10 +153,11 @@ function SliceUploader(uploader, label, data, dataSize) {
function FileUploader($scope, $q, file) {
var that = this;
$.extend(this, {
- state: 'Queued', // Queued, Uploading, Paused, Done
+ file: file,
+ locators: [],
progress: 0.0,
+ state: 'Queued', // Queued, Uploading, Paused, Done
statistics: null,
- file: file,
go: function() {
if (this._deferred)
this._deferred.reject(null, 'Restarted', null);
@@ -168,7 +180,6 @@ function FileUploader($scope, $q, file) {
},
_currentSlice: null,
_deferred: null,
- _locators: [],
$scope: $scope,
uploader: $scope.uploader,
maxBlobSize: Math.pow(2,26),
@@ -201,7 +212,7 @@ function FileUploader($scope, $q, file) {
},
onUploaderResolve: function(locator, dataSize) {
// TODO: check that._currentSlice.size == dataSize
- that._locators.push(locator);
+ that.locators.push(locator);
that._readPos += dataSize;
that.goSlice();
},
@@ -306,7 +317,7 @@ function QueueUploader($scope, $q, $timeout) {
that.reportUploadProgress);
}
}
- that.reportUploadSuccess("Done!");
+ return that.appendToCollection();
},
reportUploadError: function(jqxhr, textStatus, err) {
that.state = 'Failed';
@@ -329,6 +340,33 @@ function QueueUploader($scope, $q, $timeout) {
reportUploadProgress: function() {
// Ensure updates happen after FileUpload promise callbacks.
$timeout(function(){$scope.$apply();});
+ },
+ appendToCollection: function() {
+ var manifestText = '';
+ for (var i=0; i<$scope.uploadQueue.length; i++) {
+ var upload = $scope.uploadQueue[i];
+ manifestText += '. ' +
+ upload.locators.join(' ') +
+ ' 0:' + upload.file.size.toString() + ':' +
+ upload.file.name.replace(/ /g, '\\040') +
+ '\n';
+ }
+ return $scope.uploader.arvados.apiPromise(
+ 'collections', 'get',
+ { uuid: $scope.uuid }).
+ then(function(collection) {
+ return $scope.uploader.arvados.apiPromise(
+ 'collections', 'update',
+ { uuid: $scope.uuid,
+ collection:
+ { manifest_text:
+ '' + collection.manifest_text + manifestText }
+ }).
+ then(that.onAppendResolve);
+ }, that.onUploaderReject);
+ },
+ onAppendResolve: function() {
+ that.reportUploadSuccess("Done!");
}
});
}
diff --git a/apps/workbench/app/views/collections/_show_files.html.erb b/apps/workbench/app/views/collections/_show_files.html.erb
index 1b7ad96..7bd7eec 100644
--- a/apps/workbench/app/views/collections/_show_files.html.erb
+++ b/apps/workbench/app/views/collections/_show_files.html.erb
@@ -65,7 +65,7 @@ function unselect_all_files() {
<p>This collection is empty.</p>
<div ng-app="Workbench" class="panel panel-info">
<div class="panel-heading"><h3 class="panel-title">Upload files</h3></div>
- <div class="panel-body" ng-controller="UploadController">
+ <div class="panel-body" ng-controller="UploadController" uuid="<%= @object.uuid %>">
<div class="panel panel-primary">
<div class="panel-body">
<div class="row">
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list