[ARVADOS] updated: 6e577058389e70fd575580c3e11a122396140700
Git user
git at public.curoverse.com
Mon Apr 24 16:42:59 EDT 2017
Summary of changes:
apps/workbench/app/helpers/application_helper.rb | 2 +
apps/workbench/app/models/workflow.rb | 8 ++
.../app/views/workflows/_show_recent.html.erb | 65 ++++++++++
apps/workbench/test/integration/work_units_test.rb | 24 ++++
build/run-build-packages.sh | 2 -
build/run-library.sh | 2 +
sdk/go/crunchrunner/crunchrunner.go | 66 +++++-----
sdk/go/crunchrunner/crunchrunner_test.go | 51 ++++----
sdk/python/arvados/_ranges.py | 6 +-
sdk/python/arvados/arvfile.py | 83 ++++++++++---
sdk/python/arvados/commands/get.py | 12 +-
sdk/python/tests/test_arv_get.py | 46 +++++--
sdk/python/tests/test_arvfile.py | 134 ++++++++++++++++++++-
sdk/python/tests/test_stream.py | 7 +-
...1_add_created_by_job_task_index_to_job_tasks.rb | 5 +
...0170419173712_add_object_owner_index_to_logs.rb | 5 +
...esting_container_index_to_container_requests.rb | 5 +
services/api/db/structure.sql | 29 ++++-
services/crunch-run/logging.go | 18 +--
services/crunch-run/logging_test.go | 41 ++++++-
services/keep-web/handler.go | 14 +++
services/keep-web/handler_test.go | 31 +++++
22 files changed, 545 insertions(+), 111 deletions(-)
create mode 100644 apps/workbench/app/views/workflows/_show_recent.html.erb
create mode 100644 services/api/db/migrate/20170419173031_add_created_by_job_task_index_to_job_tasks.rb
create mode 100644 services/api/db/migrate/20170419173712_add_object_owner_index_to_logs.rb
create mode 100644 services/api/db/migrate/20170419175801_add_requesting_container_index_to_container_requests.rb
via 6e577058389e70fd575580c3e11a122396140700 (commit)
via fd65e0be3f55d48b9d573921ca0757fc51a1dd81 (commit)
via b04638275cff9b393e1bc04136d44f361b999cf8 (commit)
via c39ba5193005a4e9f619901f8348f11fada88df0 (commit)
via 2333472a4f517a227278f028bbbc4e72687c0e71 (commit)
via 5180238a10bd15302a1c15b9a428f2fdeeabdf4e (commit)
via bfc9660a8b2467893baf131b20e83e76c41ae438 (commit)
via aed7702a67426dfd9d24b512c90df8e909162179 (commit)
via 9aa83ad7b4de05dd2818885ed34111d4dcf322ea (commit)
via 3c34e713aa343d56c7cea00a9c998b06dbf411d6 (commit)
via 65e339856daf4b5c3a4a810cd3a5f1a8e386dc8c (commit)
via 151df8c3b177e4971bfbdf68c87d89599dbe0812 (commit)
via e7284afa8ccb95994dcd2009015cafc6180e7187 (commit)
via 82c40aa7d30cd8e68e2a1bdc0bd8bf03cdfea029 (commit)
via 38a4b3c43a8a6cff5a00624436b8eaa5cbfbc76b (commit)
via f79536fda9dc40f480383caa69a35663702b2ba4 (commit)
via c14ae0edcb3c386e50f46218184e8dabcbc20a37 (commit)
via 04951581a941697d68cdaf9af6661c3c412f1bce (commit)
via de99c0b2effdd43d3843f15475cee84dbde8add8 (commit)
via e3ac17f8a8aa439e21a8bf56a571f91a671313f7 (commit)
via 244c47436f294638271eef637997fd00f7ca49f5 (commit)
via 67ba19113789346005aa61d4234bc33c8677a85c (commit)
via 9609f9a5a4776671f571f765a179506d26df56da (commit)
via 03188ad6eb14ee3dcd6bdf74198624c9358936c5 (commit)
via cf311e8e16ba74467c77b5353afedc29b40a6a41 (commit)
from f1fb39153bc234b49ca77674a29abc029b12774a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6e577058389e70fd575580c3e11a122396140700
Author: radhika <radhika at curoverse.com>
Date: Mon Apr 24 16:42:20 2017 -0400
8019: add tests and address issues identified
diff --git a/services/crunch-run/logging.go b/services/crunch-run/logging.go
index a66525e..00cc06a 100644
--- a/services/crunch-run/logging.go
+++ b/services/crunch-run/logging.go
@@ -214,10 +214,10 @@ func (arvlog *ArvLogWriter) Write(p []byte) (n int, err error) {
// It has been more than throttle_period seconds since the last
// checkpoint; so reset the throttle
if arvlog.logThrottleBytesSkipped > 0 {
- arvlog.stderrBufToFlush.WriteString(fmt.Sprintf("%s Skipped %d bytes of log\n", RFC3339Timestamp(time.Now()), arvlog.logThrottleBytesSkipped))
+ arvlog.stderrBufToFlush.WriteString(fmt.Sprintf("%s Skipped %d bytes of log\n", RFC3339Timestamp(time.Now().UTC()), arvlog.logThrottleBytesSkipped))
}
- arvlog.logThrottleResetTime = time.Now().Add(time.Duration(int(crunchLogThrottlePeriod.(float64))))
+ arvlog.logThrottleResetTime = time.Now().Add(time.Second * time.Duration(int(crunchLogThrottlePeriod.(float64))))
arvlog.logThrottleBytesSoFar = 0
arvlog.logThrottleLinesSoFar = 0
arvlog.logThrottleBytesSkipped = 0
@@ -314,20 +314,24 @@ func (arvlog *ArvLogWriter) rateLimit(line []byte) (bool, []byte, error) {
}
if arvlog.bytesLogged > int64(crunchLimitLogBytesPerJob.(float64)) {
- message = fmt.Sprintf("%s Exceeded log limit %d bytes (crunch_limit_log_bytes_per_job). Log will be truncated.", RFC3339Timestamp(time.Now()), int(crunchLimitLogBytesPerJob.(float64)))
+ message = fmt.Sprintf("%s Exceeded log limit %d bytes (crunch_limit_log_bytes_per_job). Log will be truncated.", RFC3339Timestamp(time.Now().UTC()), int(crunchLimitLogBytesPerJob.(float64)))
arvlog.logThrottleResetTime = time.Now().Add(time.Duration(365 * 24 * time.Hour))
arvlog.logThrottleIsOpen = false
+
} else if arvlog.logThrottleBytesSoFar > int64(crunchLogThrottleBytes.(float64)) {
remainingTime := arvlog.logThrottleResetTime.Sub(time.Now())
- message = fmt.Sprintf("%s Exceeded rate %d bytes per %d seconds (crunch_log_throttle_bytes). Logging will be silenced for the next %d seconds.", RFC3339Timestamp(time.Now()), crunchLogThrottleBytes, int(crunchLogThrottlePeriod.(float64)), remainingTime)
+ message = fmt.Sprintf("%s Exceeded rate %d bytes per %d seconds (crunch_log_throttle_bytes). Logging will be silenced for the next %d seconds.", RFC3339Timestamp(time.Now().UTC()), int(crunchLogThrottleBytes.(float64)), int(crunchLogThrottlePeriod.(float64)), remainingTime/time.Second)
arvlog.logThrottleIsOpen = false
+
} else if arvlog.logThrottleLinesSoFar > int64(crunchLogThrottleLines.(float64)) {
remainingTime := arvlog.logThrottleResetTime.Sub(time.Now())
- message = fmt.Sprintf("%s Exceeded rate %d lines per %d seconds (crunch_log_throttle_lines), logging will be silenced for the next %d seconds.", RFC3339Timestamp(time.Now()), crunchLogThrottleLines, int(crunchLogThrottlePeriod.(float64)), remainingTime)
+ message = fmt.Sprintf("%s Exceeded rate %d lines per %d seconds (crunch_log_throttle_lines), logging will be silenced for the next %d seconds.", RFC3339Timestamp(time.Now().UTC()), int(crunchLogThrottleLines.(float64)), int(crunchLogThrottlePeriod.(float64)), remainingTime/time.Second)
arvlog.logThrottleIsOpen = false
+
} else if partialLine && arvlog.logThrottleFirstPartialLine {
arvlog.logThrottleFirstPartialLine = false
- message = fmt.Sprintf("%s Rate-limiting partial segments of long lines to one every %d seconds.", RFC3339Timestamp(time.Now()), crunchLogPartialLineThrottlePeriod)
+ message = fmt.Sprintf("%s Rate-limiting partial segments of long lines to one every %d seconds.", RFC3339Timestamp(time.Now().UTC()), int(crunchLogPartialLineThrottlePeriod.(float64)))
+
}
}
@@ -339,7 +343,7 @@ func (arvlog *ArvLogWriter) rateLimit(line []byte) (bool, []byte, error) {
if message != "" {
// Yes, write to logs, but use our "rate exceeded" message
// instead of the log message that exceeded the limit.
- message += " A complete log is still being written to Keep, and will be available when the job finishes.\n"
+ message += " A complete log is still being written to Keep, and will be available when the job finishes."
return true, []byte(message), nil
} else if partialLine {
return false, line, nil
diff --git a/services/crunch-run/logging_test.go b/services/crunch-run/logging_test.go
index 91af16d..2ea9a31 100644
--- a/services/crunch-run/logging_test.go
+++ b/services/crunch-run/logging_test.go
@@ -4,6 +4,7 @@ import (
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
. "gopkg.in/check.v1"
+ "strings"
"testing"
"time"
)
@@ -110,7 +111,12 @@ func (s *LoggingTestSuite) TestWriteMultipleLogs(c *C) {
". c556a293010069fa79a6790a931531d5+80 0:80:stdout.txt\n")
}
-func (s *LoggingTestSuite) TestWriteLogsWithRateLimit(c *C) {
+func (s *LoggingTestSuite) TestWriteLogsWithRateLimitThrottleBytes(c *C) {
+ discoveryMap["crunchLogThrottleBytes"] = float64(50)
+ defer func() {
+ discoveryMap["crunchLogThrottleBytes"] = float64(65536)
+ }()
+
api := &ArvTestClient{}
kc := &KeepTestClient{}
cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
@@ -130,6 +136,37 @@ func (s *LoggingTestSuite) TestWriteLogsWithRateLimit(c *C) {
"2015-12-29T15:51:45.000000002Z Goodbye\n"
c.Check(api.Content[0]["log"].(arvadosclient.Dict)["event_type"], Equals, "crunch-run")
- c.Check(api.Content[0]["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"], Equals, logtext)
+ stderrLog := api.Content[0]["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]
+ c.Check(true, Equals, strings.Contains(stderrLog, "Exceeded rate 50 bytes per 60 seconds"))
+ c.Check(string(kc.Content), Equals, logtext)
+}
+
+func (s *LoggingTestSuite) TestWriteLogsWithRateLimitThrottleLines(c *C) {
+ discoveryMap["crunchLogThrottleLines"] = float64(1)
+ defer func() {
+ discoveryMap["crunchLogThrottleLines"] = float64(1024)
+ }()
+
+ api := &ArvTestClient{}
+ kc := &KeepTestClient{}
+ cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzzzzzzzzzzzz")
+ cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
+
+ cr.CrunchLog.Print("Hello world!")
+ cr.CrunchLog.Print("Goodbye")
+ cr.CrunchLog.Close()
+
+ c.Check(api.Calls, Equals, 1)
+
+ mt, err := cr.LogCollection.ManifestText()
+ c.Check(err, IsNil)
+ c.Check(mt, Equals, ". 74561df9ae65ee9f35d5661d42454264+83 0:83:crunch-run.txt\n")
+
+ logtext := "2015-12-29T15:51:45.000000001Z Hello world!\n" +
+ "2015-12-29T15:51:45.000000002Z Goodbye\n"
+
+ c.Check(api.Content[0]["log"].(arvadosclient.Dict)["event_type"], Equals, "crunch-run")
+ stderrLog := api.Content[0]["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]
+ c.Check(true, Equals, strings.Contains(stderrLog, "Exceeded rate 1 lines per 60 seconds"))
c.Check(string(kc.Content), Equals, logtext)
}
commit fd65e0be3f55d48b9d573921ca0757fc51a1dd81
Merge: f1fb391 b046382
Author: radhika <radhika at curoverse.com>
Date: Mon Apr 24 10:05:32 2017 -0400
Merge branch 'master' into 8019-crunchrun-log-throttle
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list