[ARVADOS] updated: 6e38822705235d01fbb7d51626b073174a65e46e
Git user
git at public.curoverse.com
Mon Nov 21 13:39:11 EST 2016
Summary of changes:
doc/_includes/_crunch1only_begin.liquid | 2 +
doc/_includes/_crunch1only_end.liquid | 1 +
doc/_includes/_notebox_begin_warning.liquid | 2 +
doc/_includes/_pipeline_deprecation_notice.liquid | 2 +-
doc/install/install-keep-web.html.textile.liquid | 2 +-
doc/install/install-keepproxy.html.textile.liquid | 2 +-
doc/user/topics/arv-run.html.textile.liquid | 4 +
...nning-pipeline-command-line.html.textile.liquid | 4 +
...tutorial-pipeline-workbench.html.textile.liquid | 4 +
sdk/cli/bin/crunch-job | 6 +-
services/keepstore/count.go | 44 ++++++++
services/keepstore/handlers.go | 52 ++++++---
services/keepstore/keepstore.go | 3 +
services/keepstore/s3_volume.go | 125 +++++++++++++++++++--
services/keepstore/s3_volume_test.go | 30 +++++
services/keepstore/volume.go | 45 ++++++--
services/keepstore/volume_unix.go | 7 +-
17 files changed, 290 insertions(+), 45 deletions(-)
create mode 100644 doc/_includes/_crunch1only_begin.liquid
create mode 100644 doc/_includes/_crunch1only_end.liquid
create mode 100644 doc/_includes/_notebox_begin_warning.liquid
create mode 100644 services/keepstore/count.go
discards 927db6616baa78420a7ffccc59ef784fe57f6668 (commit)
discards 3080d79eb30bdb6194b3c91efbbaa4cc406eac38 (commit)
discards 3947ec4aa92c71e2402de0b34a60183ae0c52445 (commit)
discards da14fa3f2bc1b5cd2fc615b2feb5ab45f244b06a (commit)
via 6e38822705235d01fbb7d51626b073174a65e46e (commit)
via 08d5c8073523b4c5d72413d8e2095f3b68b58190 (commit)
via 7692a12d05d54f0e209dcc2ff9ae9152560c7d52 (commit)
via 3afec6824121eea6aed8c2d25567a5e8974d3100 (commit)
via e5fd39ad76ac8bcb5e7bfe0c6938fdeb7c8ab4e3 (commit)
via b1e2f45d0a926617c991410feda842a5056ff5f0 (commit)
via bf08ad601e8c69e812dcfd5fd88cd711d35647ae (commit)
via 8c716ec575b5f7679a2ab95ebea944a46ff756c1 (commit)
via d542de20d8617f5823ab8f675c114f78aaf4a924 (commit)
via 515a58c0ef8634fca2397a8609f868524a42132c (commit)
via 4568673894b4a752503ad403bd391767ac1805e5 (commit)
via b45d7c92a23390c8be246219a1c84b8736854581 (commit)
via 78889e115e6fffd5eb82e54a541bd4858f804f91 (commit)
via c1ebef70f3b66080b51ef700383f44d70736f495 (commit)
via 5977b70a38e7102a6a369074897af990944c8934 (commit)
via 1071e1163f894c2a73df76cd400d102748e5281d (commit)
via 329b27f1407d900f8de7872077e6c91ebb32107c (commit)
via 42f433ac4486c18fa6408d5f942dc394e5ff149e (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (927db6616baa78420a7ffccc59ef784fe57f6668)
\
N -- N -- N (6e38822705235d01fbb7d51626b073174a65e46e)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6e38822705235d01fbb7d51626b073174a65e46e
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 20 01:44:35 2016 -0500
10473: Tune up text-format logging.
diff --git a/services/keepstore/config.go b/services/keepstore/config.go
index 8824db9..83dd84e 100644
--- a/services/keepstore/config.go
+++ b/services/keepstore/config.go
@@ -70,6 +70,7 @@ func (cfg *Config) Start() error {
switch strings.ToLower(cfg.LogFormat) {
case "text":
log.SetFormatter(&log.TextFormatter{
+ FullTimestamp: true,
TimestampFormat: rfc3339NanoFixed,
})
case "json":
diff --git a/services/keepstore/logging_router.go b/services/keepstore/logging_router.go
index 9dfb48a..bfd006e 100644
--- a/services/keepstore/logging_router.go
+++ b/services/keepstore/logging_router.go
@@ -111,5 +111,11 @@ type loggedDuration time.Duration
// MarshalJSON formats a duration as a number of seconds, using
// fixed-point notation with no more than 6 decimal places.
func (d loggedDuration) MarshalJSON() ([]byte, error) {
- return []byte(fmt.Sprintf("%.6f", time.Duration(d).Seconds())), nil
+ return []byte(d.String()), nil
+}
+
+// String formats a duration as a number of seconds, using
+// fixed-point notation with no more than 6 decimal places.
+func (d loggedDuration) String() string {
+ return fmt.Sprintf("%.6f", time.Duration(d).Seconds())
}
commit 08d5c8073523b4c5d72413d8e2095f3b68b58190
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 20 01:39:05 2016 -0500
10473: Use fixed-point timestamps and more human-readable field ordering, hide "incoming request" if level<debug.
diff --git a/services/keepstore/logging_router.go b/services/keepstore/logging_router.go
index 30b10b3..9dfb48a 100644
--- a/services/keepstore/logging_router.go
+++ b/services/keepstore/logging_router.go
@@ -4,6 +4,8 @@ package main
// LoggingResponseWriter
import (
+ "context"
+ "fmt"
"net/http"
"strings"
"time"
@@ -65,30 +67,49 @@ type LoggingRESTRouter struct {
func (loggingRouter *LoggingRESTRouter) ServeHTTP(wrappedResp http.ResponseWriter, req *http.Request) {
tStart := time.Now()
- lgr := log.WithFields(log.Fields{
- "RequestID": loggingRouter.idGenerator.Next(),
- "RemoteAddr": req.RemoteAddr,
- "X-Forwarded-For": req.Header.Get("X-Forwarded-For"),
- "ReqMethod": req.Method,
- "ReqPath": req.URL.Path[1:],
- "ReqBytes": req.ContentLength,
+
+ // Attach a requestID-aware logger to the request context.
+ lgr := log.WithField("RequestID", loggingRouter.idGenerator.Next())
+ ctx := context.WithValue(req.Context(), "logger", lgr)
+ req = req.WithContext(ctx)
+
+ lgr = lgr.WithFields(log.Fields{
+ "remoteAddr": req.RemoteAddr,
+ "reqForwardedFor": req.Header.Get("X-Forwarded-For"),
+ "reqMethod": req.Method,
+ "reqPath": req.URL.Path[1:],
+ "reqBytes": req.ContentLength,
})
- lgr.Info("request")
+ lgr.Debug("request")
resp := LoggingResponseWriter{http.StatusOK, 0, wrappedResp, "", zeroTime}
loggingRouter.router.ServeHTTP(&resp, req)
+ tDone := time.Now()
+
statusText := http.StatusText(resp.Status)
if resp.Status >= 400 {
statusText = strings.Replace(resp.ResponseBody, "\n", "", -1)
}
+ if resp.sentHdr == zeroTime {
+ // Nobody changed status or wrote any data, i.e., we
+ // returned a 200 response with no body.
+ resp.sentHdr = tDone
+ }
- tDone := time.Now()
lgr.WithFields(log.Fields{
- "TimeTotal": tDone.Sub(tStart).Seconds(),
- "TimeToStatus": resp.sentHdr.Sub(tStart).Seconds(),
- "TimeWriteBody": tDone.Sub(resp.sentHdr).Seconds(),
- "RespStatusCode": resp.Status,
- "RespStatus": statusText,
- "RespBytes": resp.Length,
+ "timeTotal": loggedDuration(tDone.Sub(tStart)),
+ "timeToStatus": loggedDuration(resp.sentHdr.Sub(tStart)),
+ "timeWriteBody": loggedDuration(tDone.Sub(resp.sentHdr)),
+ "respStatusCode": resp.Status,
+ "respStatus": statusText,
+ "respBytes": resp.Length,
}).Info("response")
}
+
+type loggedDuration time.Duration
+
+// MarshalJSON formats a duration as a number of seconds, using
+// fixed-point notation with no more than 6 decimal places.
+func (d loggedDuration) MarshalJSON() ([]byte, error) {
+ return []byte(fmt.Sprintf("%.6f", time.Duration(d).Seconds())), nil
+}
commit 7692a12d05d54f0e209dcc2ff9ae9152560c7d52
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 20 00:30:16 2016 -0500
10473: Fixed-width timestamps.
diff --git a/services/keepstore/config.go b/services/keepstore/config.go
index b2041c3..8824db9 100644
--- a/services/keepstore/config.go
+++ b/services/keepstore/config.go
@@ -40,6 +40,8 @@ type Config struct {
var theConfig = DefaultConfig()
+const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
+
// DefaultConfig returns the default configuration.
func DefaultConfig() *Config {
return &Config{
@@ -68,11 +70,11 @@ func (cfg *Config) Start() error {
switch strings.ToLower(cfg.LogFormat) {
case "text":
log.SetFormatter(&log.TextFormatter{
- TimestampFormat: time.RFC3339Nano,
+ TimestampFormat: rfc3339NanoFixed,
})
case "json":
log.SetFormatter(&log.JSONFormatter{
- TimestampFormat: time.RFC3339Nano,
+ TimestampFormat: rfc3339NanoFixed,
})
default:
return fmt.Errorf(`unsupported log format %q (try "text" or "json")`, cfg.LogFormat)
commit 3afec6824121eea6aed8c2d25567a5e8974d3100
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Nov 18 01:49:47 2016 -0500
10473: Log requests/responses in JSON format by default; add config option for text format.
diff --git a/sdk/go/httpserver/id_generator.go b/sdk/go/httpserver/id_generator.go
new file mode 100644
index 0000000..c2830f7
--- /dev/null
+++ b/sdk/go/httpserver/id_generator.go
@@ -0,0 +1,31 @@
+package httpserver
+
+import (
+ "strconv"
+ "sync"
+ "time"
+)
+
+// IDGenerator generates alphanumeric strings suitable for use as
+// unique IDs (a given IDGenerator will never return the same ID
+// twice).
+type IDGenerator struct {
+ // Prefix is prepended to each returned ID.
+ Prefix string
+
+ lastID int64
+ mtx sync.Mutex
+}
+
+// Next returns a new ID string. It is safe to call Next from multiple
+// goroutines.
+func (g *IDGenerator) Next() string {
+ id := time.Now().UnixNano()
+ g.mtx.Lock()
+ if id <= g.lastID {
+ id = g.lastID + 1
+ }
+ g.lastID = id
+ g.mtx.Unlock()
+ return g.Prefix + strconv.FormatInt(id, 36)
+}
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 6ca31c3..43cf83a 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -8,7 +8,6 @@ import (
"fmt"
"io"
"io/ioutil"
- "log"
"net/http"
"os"
"regexp"
@@ -18,6 +17,7 @@ import (
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ log "github.com/Sirupsen/logrus"
"github.com/curoverse/azure-sdk-for-go/storage"
)
diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index d636a5e..c5dbc8f 100644
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -9,7 +9,6 @@ import (
"flag"
"fmt"
"io/ioutil"
- "log"
"math/rand"
"net"
"net/http"
@@ -22,6 +21,7 @@ import (
"testing"
"time"
+ log "github.com/Sirupsen/logrus"
"github.com/curoverse/azure-sdk-for-go/storage"
)
diff --git a/services/keepstore/bufferpool.go b/services/keepstore/bufferpool.go
index 9a35094..38f97af 100644
--- a/services/keepstore/bufferpool.go
+++ b/services/keepstore/bufferpool.go
@@ -1,10 +1,11 @@
package main
import (
- "log"
"sync"
"sync/atomic"
"time"
+
+ log "github.com/Sirupsen/logrus"
)
type bufferPool struct {
diff --git a/services/keepstore/config.go b/services/keepstore/config.go
index dc06ef5..b2041c3 100644
--- a/services/keepstore/config.go
+++ b/services/keepstore/config.go
@@ -5,17 +5,19 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
- "log"
"strings"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ log "github.com/Sirupsen/logrus"
)
type Config struct {
Debug bool
Listen string
+ LogFormat string
+
PIDFile string
MaxBuffers int
@@ -42,6 +44,7 @@ var theConfig = DefaultConfig()
func DefaultConfig() *Config {
return &Config{
Listen: ":25107",
+ LogFormat: "json",
MaxBuffers: 128,
RequireSignatures: true,
BlobSignatureTTL: arvados.Duration(14 * 24 * time.Hour),
@@ -55,12 +58,26 @@ func DefaultConfig() *Config {
// fields, and before using the config.
func (cfg *Config) Start() error {
if cfg.Debug {
+ log.SetLevel(log.DebugLevel)
cfg.debugLogf = log.Printf
cfg.debugLogf("debugging enabled")
} else {
cfg.debugLogf = func(string, ...interface{}) {}
}
+ switch strings.ToLower(cfg.LogFormat) {
+ case "text":
+ log.SetFormatter(&log.TextFormatter{
+ TimestampFormat: time.RFC3339Nano,
+ })
+ case "json":
+ log.SetFormatter(&log.JSONFormatter{
+ TimestampFormat: time.RFC3339Nano,
+ })
+ default:
+ return fmt.Errorf(`unsupported log format %q (try "text" or "json")`, cfg.LogFormat)
+ }
+
if cfg.MaxBuffers < 0 {
return fmt.Errorf("MaxBuffers must be greater than zero")
}
diff --git a/services/keepstore/config_test.go b/services/keepstore/config_test.go
index eaa0904..a6d46e5 100644
--- a/services/keepstore/config_test.go
+++ b/services/keepstore/config_test.go
@@ -1,7 +1,7 @@
package main
import (
- "log"
+ log "github.com/Sirupsen/logrus"
)
func init() {
diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index 9708b4e..40b4839 100644
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -958,7 +958,7 @@ func TestGetHandlerClientDisconnect(t *testing.T) {
ok := make(chan struct{})
go func() {
req, _ := http.NewRequest("GET", fmt.Sprintf("/%s+%d", TestHash, len(TestBlock)), nil)
- (&LoggingRESTRouter{MakeRESTRouter()}).ServeHTTP(resp, req)
+ (&LoggingRESTRouter{router: MakeRESTRouter()}).ServeHTTP(resp, req)
ok <- struct{}{}
}()
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index b51009e..ed0d8d7 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -15,7 +15,6 @@ import (
"fmt"
"github.com/gorilla/mux"
"io"
- "log"
"net/http"
"os"
"regexp"
@@ -24,6 +23,8 @@ import (
"strings"
"sync"
"time"
+
+ log "github.com/Sirupsen/logrus"
)
// MakeRESTRouter returns a new mux.Router that forwards all Keep
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index 4eaaea8..d82c7d0 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -3,7 +3,6 @@ package main
import (
"flag"
"fmt"
- "log"
"net"
"net/http"
"os"
@@ -15,6 +14,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/config"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
+ log "github.com/Sirupsen/logrus"
"github.com/coreos/go-systemd/daemon"
"github.com/ghodss/yaml"
)
@@ -151,8 +151,7 @@ func main() {
// Middleware stack: logger, MaxRequests limiter, method handlers
http.Handle("/", &LoggingRESTRouter{
- httpserver.NewRequestLimiter(theConfig.MaxRequests,
- MakeRESTRouter()),
+ router: httpserver.NewRequestLimiter(theConfig.MaxRequests, MakeRESTRouter()),
})
// Set up a TCP listener.
diff --git a/services/keepstore/logging_router.go b/services/keepstore/logging_router.go
index 0f556b5..30b10b3 100644
--- a/services/keepstore/logging_router.go
+++ b/services/keepstore/logging_router.go
@@ -4,10 +4,12 @@ package main
// LoggingResponseWriter
import (
- "log"
"net/http"
"strings"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/httpserver"
+ log "github.com/Sirupsen/logrus"
)
// LoggingResponseWriter has anonymous fields ResponseWriter and ResponseBody
@@ -57,21 +59,36 @@ func (resp *LoggingResponseWriter) Write(data []byte) (int, error) {
// LoggingRESTRouter is used to add logging capabilities to mux.Router
type LoggingRESTRouter struct {
- router http.Handler
+ router http.Handler
+ idGenerator httpserver.IDGenerator
}
func (loggingRouter *LoggingRESTRouter) ServeHTTP(wrappedResp http.ResponseWriter, req *http.Request) {
- t0 := time.Now()
+ tStart := time.Now()
+ lgr := log.WithFields(log.Fields{
+ "RequestID": loggingRouter.idGenerator.Next(),
+ "RemoteAddr": req.RemoteAddr,
+ "X-Forwarded-For": req.Header.Get("X-Forwarded-For"),
+ "ReqMethod": req.Method,
+ "ReqPath": req.URL.Path[1:],
+ "ReqBytes": req.ContentLength,
+ })
+ lgr.Info("request")
+
resp := LoggingResponseWriter{http.StatusOK, 0, wrappedResp, "", zeroTime}
loggingRouter.router.ServeHTTP(&resp, req)
statusText := http.StatusText(resp.Status)
if resp.Status >= 400 {
statusText = strings.Replace(resp.ResponseBody, "\n", "", -1)
}
- now := time.Now()
- tTotal := now.Sub(t0)
- tLatency := resp.sentHdr.Sub(t0)
- tResponse := now.Sub(resp.sentHdr)
- log.Printf("[%s] %s %s %d %.6fs %.6fs %.6fs %d %d \"%s\"", req.RemoteAddr, req.Method, req.URL.Path[1:], req.ContentLength, tTotal.Seconds(), tLatency.Seconds(), tResponse.Seconds(), resp.Status, resp.Length, statusText)
+ tDone := time.Now()
+ lgr.WithFields(log.Fields{
+ "TimeTotal": tDone.Sub(tStart).Seconds(),
+ "TimeToStatus": resp.sentHdr.Sub(tStart).Seconds(),
+ "TimeWriteBody": tDone.Sub(resp.sentHdr).Seconds(),
+ "RespStatusCode": resp.Status,
+ "RespStatus": statusText,
+ "RespBytes": resp.Length,
+ }).Info("response")
}
diff --git a/services/keepstore/pull_worker.go b/services/keepstore/pull_worker.go
index 12860bb..3c6278d 100644
--- a/services/keepstore/pull_worker.go
+++ b/services/keepstore/pull_worker.go
@@ -7,8 +7,9 @@ import (
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"io"
"io/ioutil"
- "log"
"time"
+
+ log "github.com/Sirupsen/logrus"
)
// RunPullWorker is used by Keepstore to initiate pull worker channel goroutine.
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 27ac0d9..ca5b1a2 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -9,7 +9,6 @@ import (
"fmt"
"io"
"io/ioutil"
- "log"
"net/http"
"os"
"regexp"
@@ -21,6 +20,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/AdRoll/goamz/aws"
"github.com/AdRoll/goamz/s3"
+ log "github.com/Sirupsen/logrus"
)
const (
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index 10e9158..6389d50 100644
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -7,13 +7,13 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
- "log"
"os"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"github.com/AdRoll/goamz/s3"
"github.com/AdRoll/goamz/s3/s3test"
+ log "github.com/Sirupsen/logrus"
check "gopkg.in/check.v1"
)
diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go
index 27d6216..696c3e5 100644
--- a/services/keepstore/trash_worker.go
+++ b/services/keepstore/trash_worker.go
@@ -2,10 +2,10 @@ package main
import (
"errors"
- "log"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ log "github.com/Sirupsen/logrus"
)
// RunTrashWorker is used by Keepstore to initiate trash worker channel goroutine.
diff --git a/services/keepstore/usage.go b/services/keepstore/usage.go
index 29f89f5..887cfd3 100644
--- a/services/keepstore/usage.go
+++ b/services/keepstore/usage.go
@@ -48,6 +48,10 @@ Listen:
"address" is a host IP address or name and "port" is a port number
or name.
+LogFormat:
+
+ Format of request/response and error logs: "json" or "text".
+
PIDFile:
Path to write PID file during startup. This file is kept open and
diff --git a/services/keepstore/volume_unix.go b/services/keepstore/volume_unix.go
index f9812b0..fff02aa 100644
--- a/services/keepstore/volume_unix.go
+++ b/services/keepstore/volume_unix.go
@@ -7,7 +7,6 @@ import (
"fmt"
"io"
"io/ioutil"
- "log"
"os"
"path/filepath"
"regexp"
@@ -16,6 +15,8 @@ import (
"sync"
"syscall"
"time"
+
+ log "github.com/Sirupsen/logrus"
)
type unixVolumeAdder struct {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list