[ARVADOS] updated: 244dddbbee9db8bc6fbc186462de9fe1986ea99d
git at public.curoverse.com
git at public.curoverse.com
Tue Nov 25 05:00:54 EST 2014
Summary of changes:
apps/workbench/app/assets/javascripts/event_log.js | 9 +-
.../app/assets/javascripts/pipeline_instances.js | 6 +-
.../workbench/app/assets/stylesheets/jobs.css.scss | 4 +-
apps/workbench/app/controllers/jobs_controller.rb | 14 ++-
.../controllers/pipeline_instances_controller.rb | 2 +-
apps/workbench/app/helpers/jobs_helper.rb | 23 +---
apps/workbench/app/views/jobs/push_logs.js.erb | 9 --
apps/workbench/config/routes.rb | 2 +-
.../test/integration/pipeline_instances_test.rb | 108 +++++++++++------
apps/workbench/test/integration/projects_test.rb | 37 +++++-
doc/_config.yml | 2 +-
doc/_includes/_tutorial_expectations.liquid | 4 +-
doc/api/schema/Job.html.textile.liquid | 2 +-
doc/css/bootstrap.css | 6 +-
.../create-standard-objects.html.textile.liquid | 22 +---
doc/install/install-keepproxy.html.textile.liquid | 9 +-
doc/install/install-keepstore.html.textile.liquid | 9 +-
.../install-shell-server.html.textile.liquid | 2 +-
doc/sdk/cli/index.html.textile.liquid | 6 +-
doc/sdk/cli/install.html.textile.liquid | 4 +-
doc/sdk/cli/reference.html.textile.liquid | 80 +++++++++++++
doc/sdk/cli/subcommands.html.textile.liquid | 129 ++++++++++++++++++++-
doc/sdk/python/sdk-python.html.textile.liquid | 21 +++-
.../check-environment.html.textile.liquid | 2 +-
doc/user/reference/sdk-cli.html.textile.liquid | 46 --------
.../tutorials/tutorial-keep.html.textile.liquid | 54 ++++-----
docker/api/setup.sh.in | 9 --
docker/api/supervisor.conf | 3 +
docker/build_tools/Makefile | 5 +-
sdk/cli/Gemfile.lock | 40 +++----
sdk/cli/test/test_arv-collection-create.rb | 5 +-
sdk/cli/test/test_arv-get.rb | 48 ++++++--
sdk/cli/test/test_arv-put.rb | 41 ++++++-
sdk/cli/test/test_arv-run-pipeline-instance.rb | 2 +
sdk/cli/test/test_arv-tag.rb | 6 +
sdk/python/setup.py | 32 ++---
.../api/app/controllers/application_controller.rb | 35 ++++++
.../arvados/v1/keep_disks_controller.rb | 14 +--
.../app/controllers/arvados/v1/nodes_controller.rb | 2 +-
.../controllers/arvados/v1/schema_controller.rb | 8 +-
services/api/app/models/job.rb | 8 +-
services/api/app/models/user.rb | 36 ++----
services/api/db/seeds.rb | 1 +
services/api/db/structure.sql | 2 +-
services/api/lib/current_api_client.rb | 21 ++++
services/api/lib/simulate_job_log.rb | 88 +++++++-------
services/api/lib/tasks/replay_job_log.rake | 6 +-
services/api/test/fixtures/collections.yml | 19 +++
.../test/functional/application_controller_test.rb | 44 +++++++
.../arvados/v1/collections_controller_test.rb | 6 +-
.../arvados/v1/job_reuse_controller_test.rb | 43 ++++---
.../arvados/v1/keep_disks_controller_test.rb | 31 +++--
services/api/test/unit/job_test.rb | 25 +++-
services/fuse/setup.py | 31 ++---
services/nodemanager/setup.py | 32 +++--
55 files changed, 839 insertions(+), 416 deletions(-)
delete mode 100644 apps/workbench/app/views/jobs/push_logs.js.erb
create mode 100644 doc/sdk/cli/reference.html.textile.liquid
delete mode 100644 doc/user/reference/sdk-cli.html.textile.liquid
via 244dddbbee9db8bc6fbc186462de9fe1986ea99d (commit)
via cbd9b9702b30ff610403c3f8630144630cdaafba (commit)
via 5aecf0f1c591bc1f473b6bb56715ce8db5013268 (commit)
via f8991d36180bf9c2c261dee519b0c878306a35d5 (commit)
via 7b8db198ad4cf91e605f099f78f5c4a1bef152ff (commit)
via 257ecfece0f6941011c85e735459d86b9f850d25 (commit)
via 333402104e6b7a163bf3f8483a928dbe571b5c2c (commit)
via 60cf64002cee6af43fe8b6a122c104a12c1fd7bf (commit)
via 1a4282c6d30b209a882c255e0d5777851ff6f034 (commit)
via f3d43ab311114a7c25b7ecc47f63affdc7197efb (commit)
via a8c9797de0fac6cc28d04daeade83d5e0c558076 (commit)
via aa1069044b7dbbba7d5fc302adfe7fc50efe9472 (commit)
via 399a90e3372ce5255231179777190770934e585a (commit)
via e828f379a48ac0c60794f6e4c2969158afcb4f48 (commit)
via ca493dcca8463dc5976b31de0b0dfed3c4d26d9b (commit)
via 7a503f8bceb9969d41116ed04f1b9edf53c169c2 (commit)
via 34609757e88667be13d2e8294a2cfbd4c5cccfac (commit)
via efbf7dad04f239fd83ed477a3b9a1cf5b5b979aa (commit)
via a34aaf6d03786c865d327cf16537ddf3ff221b3a (commit)
via b9d52229719c49e13aa8dfaae69da88a23fbfe44 (commit)
via 4c7615263f17507405895ceb906f7ecae0658d1d (commit)
via ac4cdfc2577b9d25ccbc9ac5d8f0333a81102367 (commit)
via 2cc7a4873db5faa8792ff2bd30fca1b8596d903c (commit)
via 93768ec1fae2054e22d2b7a290ae7f820b8ff77a (commit)
via 8fc4de54ff2845d3886130cfae33a0e47894a011 (commit)
via ddaad003304ec76d7ebd124389fc41f1a612c997 (commit)
via c431fce8f7a92d347c54b4531400efe1f7cf033b (commit)
via e89335a1e4f6c0c0898f941e2ce0dd160b16ead9 (commit)
via c9760bc4b644c0aa877fb4137d2f6eac585ed945 (commit)
via 5b3a6fbc6d91db24dff4b8b3a0edf0b7a35039ab (commit)
via 7910f3344a9d67e66d2fbbfaf030305717740cff (commit)
via 6ebc305620a6cabe8e5c49db94247e1177c73d36 (commit)
via 959c8a9e071287584bfb7267c5a9b299038ed039 (commit)
via ba251e274d684688217d0b1f8b1c8b14ce13feaf (commit)
via 225ed58387bcf549c99bf1da94174bf0cf9df6b7 (commit)
via b93f097bb9406f5b3378866f9cb9796da9dab063 (commit)
via 55cf82d6111afcbb54714fbb4be9f1ffe642d6ef (commit)
via 38243ab150194fbdfc273ef599e77ac86d27e7fc (commit)
via 9165c95ead28f64f2337a5210cedc2bda393e9e5 (commit)
via 5c7961cabd0e7d9861a735bfe0958909ae2fdb14 (commit)
via 855ba975a64411ec8ed36ddb1a52a46f0b03cb10 (commit)
via 3dc42f2d801a6bf0d752091e2ae3e4d81180467f (commit)
via 41be6cd19c17e8da5663a4dbeec9dfea2d0fec70 (commit)
via f951ebe0571f88518f239f9ed2a83da9a13b4a8b (commit)
via c5fd3fb6be57c79f7a55e0fe6a1d3f329d82a7b5 (commit)
via 4143b4814ff468765dffe8037567cb2dd54fd1af (commit)
via c33f4e367c8101c70c31645ede4ac12ce85947c9 (commit)
via de758e3ecf76410ec43ad9f39fd9dbe750de637d (commit)
via 33c4a5ab355904e900f72fb5d5a16396323ef7ec (commit)
via 1a2e18ce382675b73fd6f8d54eea29bf47f26bb3 (commit)
via d6db40a49670b9d5612a6012c0e33639c1fabd4c (commit)
via a63d484f26c644069a14434d8a5bfc16d95eb147 (commit)
via a4e5662ecb63c3b76c3538529dc02fafa3169d19 (commit)
via 992ddfa686f54799e0cee1690e7c59ceb98977f9 (commit)
via 48ccafc46692ae60300039937a15c0ec6f106697 (commit)
via 5185c0d7e1d9aa8ea4abe46e4bbc0927e52e9a66 (commit)
via 5c375252581488d15345f0dbd8aba7dd3cfa6977 (commit)
via b4453b1c8d7b374a07f740c29613f34c92d9a314 (commit)
via 09930fe84ce46936fb1cd0313b12fabddcbc4136 (commit)
via 86eafc4c929cd5e7f34de22bef4dfe0a5fc3c97c (commit)
via d6e63c5fb38996fc0ee460128269e528a5775d80 (commit)
via beb7bc17e3449280e1e8ad85282aeb4afdda7325 (commit)
via 56a7b681136be084efad5d308d2db9bceb1eb95e (commit)
via 6e49612afcbdf7422b86f68e0d90b3ec36c77e07 (commit)
via 9893a0fde39400fcf4672fff856051ac52bf31f2 (commit)
via 57813e0183970d5f37b6e7ad4ded517bc35c98e6 (commit)
via d677ef9c9b8dda9592be9e350c71e76058fc2c57 (commit)
via 4c138e32f5c8c565a2031017e26827d37688b037 (commit)
via 7533ea385aaeb69e7b8b71d1affc955d463f991c (commit)
via 41f2b53c35ec79498a513cce2e2fd9019e769c70 (commit)
via b709c1bdaefdbbb8819cfd497fa557ef2d662a9c (commit)
via 5662a056ff6e3328966272a4a7852d0d3f572739 (commit)
from 45e0087ddae93d37206e539296e2ccd84391d46b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 244dddbbee9db8bc6fbc186462de9fe1986ea99d
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Tue Nov 25 10:36:07 2014 +0100
4233: prepend historical log entries
diff --git a/apps/workbench/app/assets/javascripts/pipeline_instances.js b/apps/workbench/app/assets/javascripts/pipeline_instances.js
index bd87b28..102c602 100644
--- a/apps/workbench/app/assets/javascripts/pipeline_instances.js
+++ b/apps/workbench/app/assets/javascripts/pipeline_instances.js
@@ -98,7 +98,11 @@ $(document).on('arv-log-event', '.arv-log-event-handler-append-logs', function(e
var wasatbottom = ($(this).scrollTop() + $(this).height() >= this.scrollHeight);
if (eventData.event_type == "stderr" || eventData.event_type == "stdout") {
- $(this).append(eventData.properties.text);
+ if( eventData.prepend ) {
+ $(this).prepend(eventData.properties.text);
+ } else {
+ $(this).append(eventData.properties.text);
+ }
}
if (wasatbottom) {
diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index b1ef33a..08fb94d 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -64,7 +64,8 @@ class JobsController < ApplicationController
["object_uuid", "in", [@object.uuid]]])
.limit(500)
.results
- .reverse
+ .to_a
+ .map{ |e| e.serializable_hash.merge({ 'prepend' => true }) }
respond_to do |format|
format.json { render json: @logs }
end
commit cbd9b9702b30ff610403c3f8630144630cdaafba
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Tue Nov 25 10:00:51 2014 +0100
4233: use "success" JS function after AJAX call to get logs rather than a JS view to "push" logs to the log chart
diff --git a/apps/workbench/app/assets/javascripts/event_log.js b/apps/workbench/app/assets/javascripts/event_log.js
index e6c18c5..a3dbe53 100644
--- a/apps/workbench/app/assets/javascripts/event_log.js
+++ b/apps/workbench/app/assets/javascripts/event_log.js
@@ -339,9 +339,16 @@ $(document).on('ready ajax:complete', function() {
window.redraw = false;
createJobGraph($(graph_div).attr('id'));
+ var object_uuid = $(graph_div).data('object-uuid');
+ // if there are any listeners for this object uuid or "all", we will trigger the event
+ var matches = ".arv-log-event-listener[data-object-uuid=\"" + object_uuid + "\"],.arv-log-event-listener[data-object-uuids~=\"" + object_uuid + "\"]";
$(document).trigger('ajax:send');
- $.get('/jobs/' + $(graph_div).data('object-uuid') + '/push_logs.js');
+ $.get('/jobs/' + $(graph_div).data('object-uuid') + '/logs.json', function(data) {
+ data.forEach( function( entry ) {
+ $(matches).trigger('arv-log-event', entry);
+ });
+ });
setInterval( function() {
if( recreate ) {
diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index e6d0ccd..b1ef33a 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -57,8 +57,17 @@ class JobsController < ApplicationController
super
end
- def push_logs
- @push_logs = stderr_log_records([@object.uuid])
+ def logs
+ @logs = Log.select(%w(event_type object_uuid event_at properties))
+ .order('event_at DESC')
+ .filter([["event_type", "=", "stderr"],
+ ["object_uuid", "in", [@object.uuid]]])
+ .limit(500)
+ .results
+ .reverse
+ respond_to do |format|
+ format.json { render json: @logs }
+ end
end
def index_pane_list
diff --git a/apps/workbench/app/helpers/jobs_helper.rb b/apps/workbench/app/helpers/jobs_helper.rb
index b037267..06c3d0d 100644
--- a/apps/workbench/app/helpers/jobs_helper.rb
+++ b/apps/workbench/app/helpers/jobs_helper.rb
@@ -2,9 +2,10 @@ module JobsHelper
def stderr_log_history(job_uuids)
results = []
- log_history = stderr_log_records(job_uuids)
- if !log_history.empty?
- reversed_results = log_history.reverse
+ log_history = Log.where(event_type: 'stderr',
+ object_uuid: job_uuids).order('id DESC')
+ if !log_history.results.empty?
+ reversed_results = log_history.results.reverse
reversed_results.each do |entry|
if entry.andand.properties
properties = entry.properties
@@ -18,20 +19,4 @@ module JobsHelper
return results
end
- def stderr_log_records(job_uuids, extra_filters = nil)
- filters = [["event_type", "=", "stderr"],
- ["object_uuid", "in", job_uuids]]
- filters += extra_filters if extra_filters
- last_entry = Log.select(%w(event_at)).order('id DESC').limit(1).filter(filters).results.first
- if last_entry
- filters += [["event_at", ">=", last_entry.event_at - 10.minutes]]
- Log.select(%w(event_type object_uuid event_at properties))
- .order('id DESC')
- .filter(filters)
- .results
- else
- []
- end
- end
-
end
diff --git a/apps/workbench/app/views/jobs/push_logs.js.erb b/apps/workbench/app/views/jobs/push_logs.js.erb
deleted file mode 100644
index a811ed6..0000000
--- a/apps/workbench/app/views/jobs/push_logs.js.erb
+++ /dev/null
@@ -1,9 +0,0 @@
-var object_uuid = '<%= @object.uuid %>';
-// if there are any listeners for this object uuid or "all", we will trigger the event
-var matches = ".arv-log-event-listener[data-object-uuid=\"" + object_uuid + "\"],.arv-log-event-listener[data-object-uuids~=\"" + object_uuid + "\"],.arv-log-event-listener[data-object-uuid=\"all\"],.arv-log-event-listener[data-object-kind=\"" + parsedData.object_kind + "\"]";
-<% @push_logs.each do |log_record| %>
- var parsedData = { object_uuid: '<%=j log_record.object_uuid %>',
- event_type: '<%=j log_record.event_type %>',
- properties: { text: '<%=j log_record.properties[:text] %>' } };
- $(matches).trigger('arv-log-event', parsedData);
-<% end %>
\ No newline at end of file
diff --git a/apps/workbench/config/routes.rb b/apps/workbench/config/routes.rb
index 820303d..86cdc38 100644
--- a/apps/workbench/config/routes.rb
+++ b/apps/workbench/config/routes.rb
@@ -22,7 +22,7 @@ ArvadosWorkbench::Application.routes.draw do
resources :job_tasks
resources :jobs do
post 'cancel', :on => :member
- get 'push_logs', :on => :member
+ get 'logs', :on => :member
end
match '/logout' => 'sessions#destroy', via: [:get, :post]
get '/logged_out' => 'sessions#index'
commit 5aecf0f1c591bc1f473b6bb56715ce8db5013268
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Tue Nov 25 08:41:37 2014 +0100
4233: convert accidental tabs to spaces
diff --git a/apps/workbench/app/assets/stylesheets/jobs.css.scss b/apps/workbench/app/assets/stylesheets/jobs.css.scss
index 565c354..cf93f20 100644
--- a/apps/workbench/app/assets/stylesheets/jobs.css.scss
+++ b/apps/workbench/app/assets/stylesheets/jobs.css.scss
@@ -10,6 +10,6 @@
}
.morris-hover-point {
- text-align: left;
- width: 100%;
+ text-align: left;
+ width: 100%;
}
\ No newline at end of file
diff --git a/services/api/lib/simulate_job_log.rb b/services/api/lib/simulate_job_log.rb
index 71c9825..fc124c8 100644
--- a/services/api/lib/simulate_job_log.rb
+++ b/services/api/lib/simulate_job_log.rb
@@ -1,51 +1,49 @@
module SimulateJobLog
- def replay(filename, multiplier = 1, simulated_job_uuid = nil)
- raise "Environment must be development or test" unless [ 'test', 'development' ].include? ENV['RAILS_ENV']
+ def replay(filename, multiplier = 1, simulated_job_uuid = nil)
+ raise "Environment must be development or test" unless [ 'test', 'development' ].include? ENV['RAILS_ENV']
- multiplier = multiplier.to_f
- multiplier = 1.0 if multiplier <= 0
+ multiplier = multiplier.to_f
+ multiplier = 1.0 if multiplier <= 0
- actual_start_time = Time.now
- log_start_time = nil
+ actual_start_time = Time.now
+ log_start_time = nil
- act_as_system_user do
- File.open(filename).each.with_index do |line, index|
- cols = {}
- cols[:timestamp], rest_of_line = line.split(' ', 2)
- begin
- cols[:timestamp] = Time.strptime( cols[:timestamp], "%Y-%m-%d_%H:%M:%S" )
- rescue ArgumentError
- if line =~ /^((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d\d:\d\d:\d\d \d{4}) (.*)/
- # Wed Nov 19 07:12:39 2014
- cols[:timestamp] = Time.strptime( $1, "%a %b %d %H:%M:%S %Y" )
- rest_of_line = $2
- else
- STDERR.puts "Ignoring log line because of unknown time format: #{line}"
- end
- end
- cols[:job_uuid], cols[:pid], cols[:task], cols[:event_type], cols[:message] = rest_of_line.split(' ', 5)
- # Override job uuid with a simulated one if specified
- cols[:job_uuid] = simulated_job_uuid || cols[:job_uuid]
- # determine when we want to simulate this log being created, based on the time multiplier
- log_start_time = cols[:timestamp] if log_start_time.nil?
- log_time = cols[:timestamp]
- actual_elapsed_time = Time.now - actual_start_time
- log_elapsed_time = log_time - log_start_time
- modified_elapsed_time = log_elapsed_time / multiplier
- pause_time = modified_elapsed_time - actual_elapsed_time
- if pause_time > 0
- sleep pause_time
- end
- # output log entry for debugging and create it in the current environment's database
- puts "#{index} #{cols.to_yaml}\n"
- Log.new({
- event_at: Time.zone.local_to_utc(cols[:timestamp]),
- object_uuid: cols[:job_uuid],
- event_type: cols[:event_type],
- properties: { 'text' => line }
- }).save!
- end
- end
+ act_as_system_user do
+ File.open(filename).each.with_index do |line, index|
+ cols = {}
+ cols[:timestamp], rest_of_line = line.split(' ', 2)
+ begin
+ cols[:timestamp] = Time.strptime( cols[:timestamp], "%Y-%m-%d_%H:%M:%S" )
+ rescue ArgumentError
+ if line =~ /^((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d\d:\d\d:\d\d \d{4}) (.*)/
+ # Wed Nov 19 07:12:39 2014
+ cols[:timestamp] = Time.strptime( $1, "%a %b %d %H:%M:%S %Y" )
+ rest_of_line = $2
+ else
+ STDERR.puts "Ignoring log line because of unknown time format: #{line}"
+ end
+ end
+ cols[:job_uuid], cols[:pid], cols[:task], cols[:event_type], cols[:message] = rest_of_line.split(' ', 5)
+ # Override job uuid with a simulated one if specified
+ cols[:job_uuid] = simulated_job_uuid || cols[:job_uuid]
+ # determine when we want to simulate this log being created, based on the time multiplier
+ log_start_time = cols[:timestamp] if log_start_time.nil?
+ log_time = cols[:timestamp]
+ actual_elapsed_time = Time.now - actual_start_time
+ log_elapsed_time = log_time - log_start_time
+ modified_elapsed_time = log_elapsed_time / multiplier
+ pause_time = modified_elapsed_time - actual_elapsed_time
+ sleep pause_time if pause_time > 0
+ # output log entry for debugging and create it in the current environment's database
+ puts "#{index} #{cols.to_yaml}\n"
+ Log.new({
+ event_at: Time.zone.local_to_utc(cols[:timestamp]),
+ object_uuid: cols[:job_uuid],
+ event_type: cols[:event_type],
+ properties: { 'text' => line }
+ }).save!
+ end
+ end
- end
+ end
end
diff --git a/services/api/lib/tasks/replay_job_log.rake b/services/api/lib/tasks/replay_job_log.rake
index fad205b..14aa3be 100644
--- a/services/api/lib/tasks/replay_job_log.rake
+++ b/services/api/lib/tasks/replay_job_log.rake
@@ -1,7 +1,7 @@
require 'simulate_job_log'
desc 'Simulate job logging from a file. Three arguments: log filename, time multipler (optional), simulated job uuid (optional). E.g. (use quotation marks if using spaces between args): rake "replay_job_log[log.txt, 2.0, qr1hi-8i9sb-nf3qk0xzwwz3lre]"'
task :replay_job_log, [:filename, :multiplier, :uuid] => :environment do |t, args|
- include SimulateJobLog
- abort("No filename specified.") if args[:filename].blank?
- replay( args[:filename], args[:multiplier].to_f, args[:uuid] )
+ include SimulateJobLog
+ abort("No filename specified.") if args[:filename].blank?
+ replay( args[:filename], args[:multiplier].to_f, args[:uuid] )
end
commit f8991d36180bf9c2c261dee519b0c878306a35d5
Merge: 45e0087 7b8db19
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Tue Nov 25 08:30:20 2014 +0100
Merge branch 'master' into 4233-graph-job-stats
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list