[ARVADOS] updated: 1.2.0-116-g5d6a2c2e4
Git user
git at public.curoverse.com
Wed Sep 26 16:19:26 EDT 2018
Summary of changes:
.../controllers/arvados/v1/schema_controller.rb | 2 ++
services/api/config/application.default.yml | 11 ++++++++++
services/crunch-run/crunchrun.go | 25 +++++++++++++++-------
services/crunch-run/logging.go | 8 +++----
services/crunch-run/logging_test.go | 16 +++++++-------
5 files changed, 42 insertions(+), 20 deletions(-)
via 5d6a2c2e4b85434e9bae1dd0adc27c284cb9ea85 (commit)
from d5aeb0c9a768004dfb806678573f11b5048e26f0 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5d6a2c2e4b85434e9bae1dd0adc27c284cb9ea85
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Sep 26 16:19:16 2018 -0400
10181: Rename configs, add to API server config, support SIGUSR1.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 49fc398e1..2d0bc114f 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -58,6 +58,8 @@ class Arvados::V1::SchemaController < ApplicationController
crunchLogThrottleLines: Rails.application.config.crunch_log_throttle_lines,
crunchLimitLogBytesPerJob: Rails.application.config.crunch_limit_log_bytes_per_job,
crunchLogPartialLineThrottlePeriod: Rails.application.config.crunch_log_partial_line_throttle_period,
+ crunchLogUpdatePeriod: Rails.application.config.crunch_log_update_period,
+ crunchLogUpdateSize: Rails.application.config.crunch_log_update_size,
remoteHosts: Rails.configuration.remote_hosts,
remoteHostsViaDNS: Rails.configuration.remote_hosts_via_dns,
websocketUrl: Rails.application.config.websocket_address,
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index a76a567ee..4aa21733c 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -279,6 +279,17 @@ common:
crunch_log_partial_line_throttle_period: 5
+ # Container logs are written to Keep and saved in a collection,
+ # which is updated periodically while the container runs. This
+ # value sets the interval (given in seconds) between collection
+ # updates.
+ crunch_log_update_period: 1800
+
+ # The log collection is also updated when the specified amount of
+ # log data (given in bytes) is produced in less than one update
+ # period.
+ crunch_log_update_size: 33554432
+
# Attributes to suppress in events and audit logs. Notably,
# specifying ["manifest_text"] here typically makes the database
# smaller and faster.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 157cc8184..e4aacf56d 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -1177,14 +1177,23 @@ func (runner *ContainerRunner) WaitFinish() error {
}
}
-func (runner *ContainerRunner) checkpointLogs() {
- ticker := time.NewTicker(crunchLogCheckpointMaxDuration / 360)
+func (runner *ContainerRunner) updateLogs() {
+ ticker := time.NewTicker(crunchLogUpdatePeriod / 360)
defer ticker.Stop()
- saveAtTime := time.Now().Add(crunchLogCheckpointMaxDuration)
- saveAtSize := crunchLogCheckpointMaxBytes
+ sigusr1 := make(chan os.Signal, 1)
+ signal.Notify(sigusr1, syscall.SIGUSR1)
+ defer signal.Stop(sigusr1)
+
+ saveAtTime := time.Now().Add(crunchLogUpdatePeriod)
+ saveAtSize := crunchLogUpdateSize
var savedSize int64
- for range ticker.C {
+ for {
+ select {
+ case <-ticker.C:
+ case <-sigusr1:
+ saveAtTime = time.Now()
+ }
runner.logMtx.Lock()
done := runner.LogsPDH != nil
runner.logMtx.Unlock()
@@ -1195,8 +1204,8 @@ func (runner *ContainerRunner) checkpointLogs() {
if size == savedSize || (time.Now().Before(saveAtTime) && size < saveAtSize) {
continue
}
- saveAtTime = time.Now().Add(crunchLogCheckpointMaxDuration)
- saveAtSize = runner.LogCollection.Size() + crunchLogCheckpointMaxBytes
+ saveAtTime = time.Now().Add(crunchLogUpdatePeriod)
+ saveAtSize = runner.LogCollection.Size() + crunchLogUpdateSize
saved, err := runner.saveLogCollection()
if err != nil {
runner.CrunchLog.Printf("error updating log collection: %s", err)
@@ -1690,7 +1699,7 @@ func NewContainerRunner(client *arvados.Client, api IArvadosClient, kc IKeepClie
cr.CrunchLog.Immediate = log.New(os.Stderr, containerUUID+" ", 0)
loadLogThrottleParams(api)
- go cr.checkpointLogs()
+ go cr.updateLogs()
return cr, nil
}
diff --git a/services/crunch-run/logging.go b/services/crunch-run/logging.go
index fc9c6f628..f8ddd563c 100644
--- a/services/crunch-run/logging.go
+++ b/services/crunch-run/logging.go
@@ -197,8 +197,8 @@ var crunchLogThrottleLines int64 = 1024
var crunchLogPartialLineThrottlePeriod time.Duration = time.Second * 5
var crunchLogBytesPerEvent int64 = 4096
var crunchLogSecondsBetweenEvents = time.Second
-var crunchLogCheckpointMaxDuration = time.Hour / 2
-var crunchLogCheckpointMaxBytes = int64(1 << 25)
+var crunchLogUpdatePeriod = time.Hour / 2
+var crunchLogUpdateSize = int64(1 << 25)
// ArvLogWriter is an io.WriteCloser that processes each write by
// writing it through to another io.WriteCloser (typically a
@@ -401,7 +401,7 @@ func loadLogThrottleParams(clnt IArvadosClient) {
loadDuration(&crunchLogPartialLineThrottlePeriod, "crunchLogPartialLineThrottlePeriod")
loadInt64(&crunchLogBytesPerEvent, "crunchLogBytesPerEvent")
loadDuration(&crunchLogSecondsBetweenEvents, "crunchLogSecondsBetweenEvents")
- loadInt64(&crunchLogCheckpointMaxBytes, "crunchLogCheckpointMaxBytes")
- loadDuration(&crunchLogCheckpointMaxDuration, "crunchLogCheckpointMaxDuration")
+ loadInt64(&crunchLogUpdateSize, "crunchLogUpdateSize")
+ loadDuration(&crunchLogUpdatePeriod, "crunchLogUpdatePeriod")
}
diff --git a/services/crunch-run/logging_test.go b/services/crunch-run/logging_test.go
index 64fb791a5..78f984db5 100644
--- a/services/crunch-run/logging_test.go
+++ b/services/crunch-run/logging_test.go
@@ -37,8 +37,8 @@ var _ = Suite(&LoggingTestSuite{})
func (s *LoggingTestSuite) SetUpTest(c *C) {
s.client = arvados.NewClientFromEnv()
- crunchLogCheckpointMaxDuration = time.Hour * 24 * 365
- crunchLogCheckpointMaxBytes = 1 << 50
+ crunchLogUpdatePeriod = time.Hour * 24 * 365
+ crunchLogUpdateSize = 1 << 50
}
func (s *LoggingTestSuite) TestWriteLogs(c *C) {
@@ -131,7 +131,7 @@ func (s *LoggingTestSuite) TestWriteMultipleLogs(c *C) {
c.Check(mt, Equals, ". 48f9023dc683a850b1c9b482b14c4b97+163 0:83:crunch-run.txt 83:80:stdout.txt\n")
}
-func (s *LoggingTestSuite) TestLogCheckpoint(c *C) {
+func (s *LoggingTestSuite) TestLogUpdate(c *C) {
for _, trial := range []struct {
maxBytes int64
maxDuration time.Duration
@@ -140,8 +140,8 @@ func (s *LoggingTestSuite) TestLogCheckpoint(c *C) {
{1000000, time.Millisecond},
} {
c.Logf("max %d bytes, %s", trial.maxBytes, trial.maxDuration)
- crunchLogCheckpointMaxBytes = trial.maxBytes
- crunchLogCheckpointMaxDuration = trial.maxDuration
+ crunchLogUpdateSize = trial.maxBytes
+ crunchLogUpdatePeriod = trial.maxDuration
api := &ArvTestClient{}
kc := &KeepTestClient{}
@@ -169,9 +169,9 @@ func (s *LoggingTestSuite) TestLogCheckpoint(c *C) {
mt, err := cr.LogCollection.MarshalManifest(".")
c.Check(err, IsNil)
- // Block packing depends on whether there's a
- // checkpoint between the two Goodbyes -- either way
- // the first block will be 4dc76.
+ // Block packing depends on whether there's an update
+ // between the two Goodbyes -- either way the first
+ // block will be 4dc76.
c.Check(mt, Matches, `. 4dc76e0a212bfa30c39d76d8c16da0c0\+1038 (afc503bc1b9a828b4bb543cb629e936c\+78|90699dc22545cd74a0664303f70bc05a\+39 276b49339fd5203d15a93ff3de11bfb9\+39) 0:1077:crunch-run.txt 1077:39:stdout.txt\n`)
}
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list