[ARVADOS] created: f5c622f3018c7282375f5a0dca26e64c9ea8d982
Git user
git at public.curoverse.com
Wed Feb 24 09:33:58 EST 2016
at f5c622f3018c7282375f5a0dca26e64c9ea8d982 (commit)
commit f5c622f3018c7282375f5a0dca26e64c9ea8d982
Author: radhika <radhika at curoverse.com>
Date: Wed Feb 24 09:32:19 2016 -0500
7399: throttle partial segments of long lines to once every "n" seconds
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 40c9ef6..6691683 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -338,6 +338,7 @@ common:
# "git log".
source_version: false
+ crunch_log_partial_line_throttle_period: 5
development:
force_ssl: false
diff --git a/services/api/lib/crunch_dispatch.rb b/services/api/lib/crunch_dispatch.rb
index b59279e..cbdd04e 100644
--- a/services/api/lib/crunch_dispatch.rb
+++ b/services/api/lib/crunch_dispatch.rb
@@ -429,6 +429,8 @@ class CrunchDispatch
log_throttle_bytes_so_far: 0,
log_throttle_lines_so_far: 0,
log_throttle_bytes_skipped: 0,
+ log_throttle_partial_line_last_at: 0,
+ log_throttle_first_partial_line: true,
}
i.close
@todo_job_retries.delete(job.uuid)
@@ -443,9 +445,22 @@ class CrunchDispatch
message = false
linesize = line.size
if running_job[:log_throttle_is_open]
- running_job[:log_throttle_lines_so_far] += 1
- running_job[:log_throttle_bytes_so_far] += linesize
- running_job[:bytes_logged] += linesize
+ partial_line = false
+ skip_counts = false
+ if line.start_with?('[...]') and line.end_with?('[...]')
+ partial_line = true
+ if Time.now > j[:log_throttle_partial_line_last_at] + Rails.configuration.crunch_log_partial_line_throttle_period
+ j[:log_throttle_partial_line_last_at] = Time.now
+ else
+ skip_counts = true
+ end
+ end
+
+ if !skip_counts
+ running_job[:log_throttle_lines_so_far] += 1
+ running_job[:log_throttle_bytes_so_far] += linesize
+ running_job[:bytes_logged] += linesize
+ end
if (running_job[:bytes_logged] >
Rails.configuration.crunch_limit_log_bytes_per_job)
@@ -456,14 +471,18 @@ class CrunchDispatch
elsif (running_job[:log_throttle_bytes_so_far] >
Rails.configuration.crunch_log_throttle_bytes)
remaining_time = running_job[:log_throttle_reset_time] - Time.now
- message = "Exceeded rate #{Rails.configuration.crunch_log_throttle_bytes} bytes per #{Rails.configuration.crunch_log_throttle_period} seconds (crunch_log_throttle_bytes). Logging will be silenced for the next #{remaining_time.round} seconds.\n"
+ message = "Exceeded rate #{Rails.configuration.crunch_log_throttle_bytes} bytes per #{Rails.configuration.crunch_log_throttle_period} seconds (crunch_log_throttle_bytes). Logging will be silenced for the next #{remaining_time.round} seconds."
running_job[:log_throttle_is_open] = false
elsif (running_job[:log_throttle_lines_so_far] >
Rails.configuration.crunch_log_throttle_lines)
remaining_time = running_job[:log_throttle_reset_time] - Time.now
- message = "Exceeded rate #{Rails.configuration.crunch_log_throttle_lines} lines per #{Rails.configuration.crunch_log_throttle_period} seconds (crunch_log_throttle_lines), logging will be silenced for the next #{remaining_time.round} seconds.\n"
+ message = "Exceeded rate #{Rails.configuration.crunch_log_throttle_lines} lines per #{Rails.configuration.crunch_log_throttle_period} seconds (crunch_log_throttle_lines), logging will be silenced for the next #{remaining_time.round} seconds."
running_job[:log_throttle_is_open] = false
+
+ elsif partial_line and j[:log_throttle_first_partial_line]
+ j[:log_throttle_first_partial_line] = false
+ message = "Rate-limiting partial segments of long lines to one every #{Rails.configuration.crunch_log_partial_line_throttle_period} seconds."
end
end
@@ -475,6 +494,7 @@ class CrunchDispatch
if message
# Yes, write to logs, but use our "rate exceeded" message
# instead of the log message that exceeded the limit.
+ message += " A complete log is still being written to Keep, and will be available when the job finishes.\n"
line.replace message
true
else
@@ -501,6 +521,8 @@ class CrunchDispatch
j[:log_throttle_lines_so_far] = 0
j[:log_throttle_bytes_skipped] = 0
j[:log_throttle_is_open] = true
+ j[:log_throttle_partial_line_last_at] = 0
+ j[:log_throttle_first_partial_line] = true
end
j[:buf].each do |stream, streambuf|
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list