[ARVADOS] updated: eda00143401effca0bbc0b2f6e4fde8ee2ede8a0
Git user
git at public.curoverse.com
Wed Mar 2 12:55:49 EST 2016
Summary of changes:
services/api/lib/crunch_dispatch.rb | 6 ++++--
services/api/test/unit/crunch_dispatch_test.rb | 23 ++++++++++++++---------
2 files changed, 18 insertions(+), 11 deletions(-)
via eda00143401effca0bbc0b2f6e4fde8ee2ede8a0 (commit)
from afca15637ba53e38238405afb2e14342cca33052 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit eda00143401effca0bbc0b2f6e4fde8ee2ede8a0
Author: radhika <radhika at curoverse.com>
Date: Wed Mar 2 12:55:21 2016 -0500
7399: use regexp to look for partial line segments.
diff --git a/services/api/lib/crunch_dispatch.rb b/services/api/lib/crunch_dispatch.rb
index 1017895..24d4d33 100644
--- a/services/api/lib/crunch_dispatch.rb
+++ b/services/api/lib/crunch_dispatch.rb
@@ -447,8 +447,10 @@ class CrunchDispatch
if running_job[:log_throttle_is_open]
partial_line = false
skip_counts = false
- line_splits = line.split('stderr ')
- if line_splits[1].andand.start_with?('[...]') and line_splits[1].end_with?('[...]')
+ matches = line.match(/^\S+ \S+ \d+ \d+ stderr (.*)/)
+ if matches and matches[1] and
+ (matches[1].start_with?('[...]') or matches[1].start_with?('crunchstat [...]')) and
+ matches[1].end_with?('[...]')
partial_line = true
if Time.now > running_job[:log_throttle_partial_line_last_at] + Rails.configuration.crunch_log_partial_line_throttle_period
running_job[:log_throttle_partial_line_last_at] = Time.now
diff --git a/services/api/test/unit/crunch_dispatch_test.rb b/services/api/test/unit/crunch_dispatch_test.rb
index 04b06d7..a4503a8 100644
--- a/services/api/test/unit/crunch_dispatch_test.rb
+++ b/services/api/test/unit/crunch_dispatch_test.rb
@@ -139,39 +139,44 @@ class CrunchDispatchTest < ActiveSupport::TestCase
assert_equal 1, job[:log_throttle_lines_so_far]
# first partial line segment is skipped and counted towards skipped lines
- now = Time.now
- line = "#{Time.now} stderr [...] this is first partial line segment [...]"
+ now = Time.now.strftime('%Y-%m-%d-%H:%M:%S')
+ line = "#{now} localhost 100 0 stderr [...] this is first partial line segment [...]"
limit = dispatch.rate_limit(job, line)
assert_equal true, limit
assert_includes line, "Rate-limiting partial segments of long lines", line
assert_equal 2, job[:log_throttle_lines_so_far]
# next partial line segment within throttle interval is skipped but not counted towards skipped lines
- line = "#{Time.now} stderr [...] second partial line segment within the interval [...]"
+ line = "#{now} localhost 100 0 stderr [...] second partial line segment within the interval [...]"
+ limit = dispatch.rate_limit(job, line)
+ assert_equal false, limit
+ assert_equal 2, job[:log_throttle_lines_so_far]
+
+ # crunchstat partial line segment is also skipped
+ line = "#{now} localhost 100 0 stderr crunchstat [...] second partial line segment within the interval [...]"
limit = dispatch.rate_limit(job, line)
assert_equal false, limit
assert_equal 2, job[:log_throttle_lines_so_far]
# next partial line after interval is counted towards skipped lines
sleep(1)
- line = "#{Time.now} stderr [...] third partial line segment after the interval [...]"
+ line = "#{now} localhost 100 0 stderr [...] third partial line segment after the interval [...]"
limit = dispatch.rate_limit(job, line)
assert_equal false, limit
assert_equal 3, job[:log_throttle_lines_so_far]
- now = Time.now
# this is not a valid line segment
- line = "#{now} stderr [...] does not end with [...] and is not a partial segment"
+ line = "#{now} localhost 100 0 stderr [...] does not end with [...] and is not a partial segment"
limit = dispatch.rate_limit(job, line)
assert_equal true, limit
- assert_equal "#{now} stderr [...] does not end with [...] and is not a partial segment", line
+ assert_equal "#{now} localhost 100 0 stderr [...] does not end with [...] and is not a partial segment", line
assert_equal 4, job[:log_throttle_lines_so_far]
# this also is not a valid line segment
- line = "#{now} stderr does not start correctly but ends with [...]"
+ line = "#{now} localhost 100 0 stderr does not start correctly but ends with [...]"
limit = dispatch.rate_limit(job, line)
assert_equal true, limit
- assert_equal "#{now} stderr does not start correctly but ends with [...]", line
+ assert_equal "#{now} localhost 100 0 stderr does not start correctly but ends with [...]", line
assert_equal 5, job[:log_throttle_lines_so_far]
end
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list