[ARVADOS] updated: 8e5db2221d848c35e751b932da8126b559d292f0
git at public.curoverse.com
git at public.curoverse.com
Thu May 22 22:25:07 EDT 2014
Summary of changes:
.../app/assets/javascripts/pipeline_instances.js | 15 +-
.../app/helpers/pipeline_instances_helper.rb | 13 +-
{services/keep => sdk/go}/build.sh | 9 +-
sdk/go/src/arvados.org/keepclient/hashcheck.go | 77 +++
.../src/arvados.org/keepclient/hashcheck_test.go | 85 +++
sdk/go/src/arvados.org/keepclient/keepclient.go | 206 +++++++
.../src/arvados.org/keepclient/keepclient_test.go | 677 +++++++++++++++++++++
sdk/go/src/arvados.org/keepclient/support.go | 255 ++++++++
sdk/go/src/arvados.org/streamer/streamer.go | 130 ++++
sdk/go/src/arvados.org/streamer/streamer_test.go | 366 +++++++++++
sdk/go/src/arvados.org/streamer/transfer.go | 308 ++++++++++
sdk/python/arvados/keep.py | 38 +-
sdk/python/run_test_server.py | 6 +
.../app/controllers/arvados/v1/jobs_controller.rb | 4 +-
services/api/script/crunch-dispatch.rb | 71 +--
services/api/test/fixtures/jobs.yml | 20 +
.../functional/arvados/v1/jobs_controller_test.rb | 15 +
17 files changed, 2235 insertions(+), 60 deletions(-)
copy {services/keep => sdk/go}/build.sh (85%)
create mode 100644 sdk/go/src/arvados.org/keepclient/hashcheck.go
create mode 100644 sdk/go/src/arvados.org/keepclient/hashcheck_test.go
create mode 100644 sdk/go/src/arvados.org/keepclient/keepclient.go
create mode 100644 sdk/go/src/arvados.org/keepclient/keepclient_test.go
create mode 100644 sdk/go/src/arvados.org/keepclient/support.go
create mode 100644 sdk/go/src/arvados.org/streamer/streamer.go
create mode 100644 sdk/go/src/arvados.org/streamer/streamer_test.go
create mode 100644 sdk/go/src/arvados.org/streamer/transfer.go
via 8e5db2221d848c35e751b932da8126b559d292f0 (commit)
via 43a2aef67a96033f54e9f19cd9788e059e72115a (commit)
via 2099a0a3f0ef3371ba30ed0955bbc75ffa332701 (commit)
via d5823126a51b7c31915e01fee100abe9468014e5 (commit)
via 67492abc51acc09abec9e754fc1d845679db3e87 (commit)
via 3ba5aa15ea0156a4fce63dcb43b7f972b4c760df (commit)
via d54a48fa6fe94e9b80bf32c1d357e4dc3b3d67c3 (commit)
via c6a6693dc36615effca5e3363b81199362007c59 (commit)
via 38cef39fbdfeb8176e4c755d12e43a450e868439 (commit)
via 2f32f3483d18e9a89a8b5c13e022495c8681db04 (commit)
via 3986815ae5e7e61c48f3ed979c32358710ef7e20 (commit)
via a2273675f29a0f85cb80e62b3742e82d63c365e9 (commit)
via 64aac6153e1819738d9d80e156572aeb9bf07f97 (commit)
via 941bcf698f1cfb498510a13f23d3c9d403b0435f (commit)
via 9482cf9f326ce64f9ea100aa34680278bdd6018e (commit)
via d3b11ddc2506de37b8e6538be69237d6d2a60a4a (commit)
via c3a88cbf511aa0954dac271ce6bda9c6e4f3191c (commit)
via 4ec57745d2106e955fea4442c9eccb2fce7246c4 (commit)
via 27f5c1635d56c3f3cb6c5ef069c28db939eec2a1 (commit)
via b534e4a2b167185e59ca657369fee06e5effa6cc (commit)
via e678f2a0e5d223ddd5b2a6c6a8d14a8afa6d463d (commit)
via 2b19cf9f9522dd0e8774031a54ce695e73fb72fe (commit)
via fb7f238945e33b07f1c80b0623315c1ecf86bca2 (commit)
via 09559ebbb9f52c8b8a47e21d0a9c5720f3a2b7c6 (commit)
via 99cd188a0cbd9143b690750e21abc8d8d5e6dbad (commit)
via 2a493a9215f604c63ab7bc6f0e0956d10af8ef10 (commit)
via 66d5cdb1f34d614e5ecf1da5ef6efcad3a5a51ab (commit)
via 90bf8cae5f1ba931c7cc3be7864cad8ecca1ca5e (commit)
via 33d63c6d42e824744305df3e720f8e9cbcc87d78 (commit)
via 6022ddd1b07217b9b4f31e73493fc259bd19b696 (commit)
via 28195ca73b4c86e0f17ce2db74f3d5669d344e23 (commit)
via 717bee7044a2cd44678e13e116ec1be2a83cfe60 (commit)
via 952599738f3c9f8901c895858865fc90b5e30c1d (commit)
via 6132d8efbc522b71d0084160abaaa87031678bdc (commit)
from 178f2834f7c9585485df30b1fe0e53b27e4edac5 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8e5db2221d848c35e751b932da8126b559d292f0
Merge: 43a2aef 2099a0a
Author: radhika <radhika at curoverse.com>
Date: Thu May 22 22:24:03 2014 -0400
Merge branch 'master' into 2756-eventbus-in-workbench
commit 43a2aef67a96033f54e9f19cd9788e059e72115a
Author: radhika <radhika at curoverse.com>
Date: Thu May 22 22:21:13 2014 -0400
2756: rescue any exceptions. do not write to redis. use log properties, instead of summary to store the potentially big strings.
diff --git a/apps/workbench/app/assets/javascripts/pipeline_instances.js b/apps/workbench/app/assets/javascripts/pipeline_instances.js
index ef936e3..0d83f3b 100644
--- a/apps/workbench/app/assets/javascripts/pipeline_instances.js
+++ b/apps/workbench/app/assets/javascripts/pipeline_instances.js
@@ -53,8 +53,19 @@
$(document).on('arv-log-event', '.arv-log-event-handler-append-logs', function(event, eventData){
parsedData = JSON.parse(eventData);
- summary = parsedData.summary;
- $(this).append(summary + "<br/>");
+
+ propertyText = undefined
+
+ properties = parsedData.properties;
+ if (properties !== null) {
+ propertyText = properties.text;
+ }
+
+ if (propertyText !== undefined) {
+ $(this).append(propertyText + "<br/>");
+ } else {
+ $(this).append(parsedData.summary + "<br/>");
+ }
});
})();
diff --git a/apps/workbench/app/helpers/pipeline_instances_helper.rb b/apps/workbench/app/helpers/pipeline_instances_helper.rb
index b39bfb1..56e85db 100644
--- a/apps/workbench/app/helpers/pipeline_instances_helper.rb
+++ b/apps/workbench/app/helpers/pipeline_instances_helper.rb
@@ -25,13 +25,18 @@ module PipelineInstancesHelper
def pipieline_log_history(job_uuids)
results = []
- log_history = Log.where(event_type: 'stderr', object_uuid: job_uuids).
- order('id DESC').limit(20).all
+ log_history = Log.where(event_type: 'stderr',
+ object_uuid: job_uuids).order('id DESC')
if !log_history.results.empty?
reversed_results = log_history.results.reverse
reversed_results.each do |entry|
- summary = entry.summary
- results = results.concat summary.split("\n")
+ if entry.andand.properties
+ properties = entry.properties
+ text = properties[:text]
+ if text
+ results = results.concat text.split("\n")
+ end
+ end
end
end
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 57f146c..abed9dd 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -26,12 +26,7 @@ require File.dirname(__FILE__) + '/../config/boot'
require File.dirname(__FILE__) + '/../config/environment'
require 'open3'
-$redis ||= Redis.new
-LOG_BUFFER_SIZE = 2**20
-
-$tmp_log_buffer = ''
-$previous_tmp_log_at = Time.now
-TMP_LOG_BUFFER_SIZE = 4096
+LOG_BUFFER_SIZE = 4096
class Dispatcher
include ApplicationHelper
@@ -195,9 +190,6 @@ class Dispatcher
$stderr.puts "dispatch: job #{job.uuid}"
start_banner = "dispatch: child #{t.pid} start #{Time.now.ctime.to_s}"
$stderr.puts start_banner
- $redis.set job.uuid, start_banner + "\n"
- $redis.publish job.uuid, start_banner
- $redis.publish job.owner_uuid, start_banner
@running[job.uuid] = {
stdin: i,
@@ -208,7 +200,8 @@ class Dispatcher
stderr_buf: '',
started: false,
sent_int: 0,
- job_auth: job_auth
+ job_auth: job_auth,
+ stderr_flushed_at: 0
}
i.close
end
@@ -243,33 +236,18 @@ class Dispatcher
if stderr_buf
j[:stderr_buf] << stderr_buf
- if j[:stderr_buf].index "\n"
- lines = j[:stderr_buf].lines("\n").to_a
- if j[:stderr_buf][-1] == "\n"
- j[:stderr_buf] = ''
- else
- j[:stderr_buf] = lines.pop
- end
+ if stderr_buf.index "\n" || j[:stderr_flushed_at] != Time.now.to_i
+ lines = stderr_buf.lines("\n").to_a
lines.each do |line|
$stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
$stderr.puts line
- pub_msg = "#{Time.now.ctime.to_s} #{line.strip}"
- $redis.publish job.owner_uuid, pub_msg
- $redis.publish job_uuid, pub_msg
- $redis.append job_uuid, pub_msg + "\n"
- if LOG_BUFFER_SIZE < $redis.strlen(job_uuid)
- $redis.set(job_uuid,
- $redis
- .getrange(job_uuid, (LOG_BUFFER_SIZE >> 1), -1)
- .sub(/^.*?\n/, ''))
- end
+ log_msg = "#{Time.now.ctime.to_s} #{line.strip}"
+ j[:stderr_buf] << (log_msg + " \n")
+ end
- if (TMP_LOG_BUFFER_SIZE < $tmp_log_buffer.size) || ($previous_tmp_log_at+1 < Time.now)
- $tmp_log_buffer += (pub_msg + "\n")
- write_log job_uuid
- else
- $tmp_log_buffer += (pub_msg + "\n")
- end
+ if (LOG_BUFFER_SIZE < j[:stderr_buf].size) || ((j[:stderr_flushed_at]+1) < Time.now.to_i)
+ write_log j
+ j[:stderr_flushed_at] = Time.now.to_i
end
end
end
@@ -317,7 +295,7 @@ class Dispatcher
# Ensure every last drop of stdout and stderr is consumed
read_pipes
- write_log job_done.uuid # write any remaining logs
+ write_log j_done # write any remaining logs
if j_done[:stderr_buf] and j_done[:stderr_buf] != ''
$stderr.puts j_done[:stderr_buf] + "\n"
@@ -346,8 +324,6 @@ class Dispatcher
# Invalidate the per-job auth token
j_done[:job_auth].update_attributes expires_at: Time.now
- $redis.publish job_done.uuid, "end"
-
@running.delete job_done.uuid
end
@@ -415,17 +391,22 @@ class Dispatcher
end
# send message to log table. we want these records to be transient
- def write_log job_uuid
- if $tmp_log_buffer == ''
- return
+ def write_log running_job
+ begin
+ if (running_job && running_job[:stderr_buf] != '')
+ log = Log.new(object_uuid: running_job[:job].uuid,
+ event_type: 'stderr',
+ properties: {"text" => running_job[:stderr_buf]})
+ log.save!
+ running_job[:stderr_buf] = ''
+ running_job[:stderr_flushed_at] = Time.now.to_i
+ end
+ rescue
+ running_job[:stderr_buf] = 'Failed to write logs \n'
+ running_job[:stderr_flushed_at] = Time.now.to_i
end
- log = Log.new(object_uuid: job_uuid,
- event_type:'stderr',
- summary: $tmp_log_buffer)
- log.save!
- $tmp_log_buffer = ''
- $previous_tmp_log_at = Time.now
end
+
end
# This is how crunch-job child procs know where the "refresh" trigger file is
commit 67492abc51acc09abec9e754fc1d845679db3e87
Author: radhika <radhika at curoverse.com>
Date: Thu May 22 13:35:27 2014 -0400
2756: use stderr for event_type instead of transient-log-entry
diff --git a/apps/workbench/app/helpers/pipeline_instances_helper.rb b/apps/workbench/app/helpers/pipeline_instances_helper.rb
index 5a163c4..b39bfb1 100644
--- a/apps/workbench/app/helpers/pipeline_instances_helper.rb
+++ b/apps/workbench/app/helpers/pipeline_instances_helper.rb
@@ -25,8 +25,8 @@ module PipelineInstancesHelper
def pipieline_log_history(job_uuids)
results = []
- log_history = Log.where(event_type: 'transient-log-entry',
- object_uuid: job_uuids).order('id DESC').limit(20).all
+ log_history = Log.where(event_type: 'stderr', object_uuid: job_uuids).
+ order('id DESC').limit(20).all
if !log_history.results.empty?
reversed_results = log_history.results.reverse
reversed_results.each do |entry|
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 5cf7cac..57f146c 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -420,7 +420,7 @@ class Dispatcher
return
end
log = Log.new(object_uuid: job_uuid,
- event_type:'transient-log-entry',
+ event_type:'stderr',
summary: $tmp_log_buffer)
log.save!
$tmp_log_buffer = ''
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list