[ARVADOS] updated: 767e1199d0e1bdf2b564b5c58a91d29141eb67d7
git at public.curoverse.com
git at public.curoverse.com
Tue Nov 3 23:24:05 EST 2015
Summary of changes:
backports/python-ciso8601/fpm-info.sh | 3 +
backports/python-llfuse/fpm-info.sh | 3 +
backports/python-pycrypto/fpm-info.sh | 3 +
backports/python-pycurl/fpm-info.sh | 3 +
doc/install/install-keep-web.html.textile.liquid | 4 +
sdk/cwl/arvados_cwl/__init__.py | 98 +++++++++++++++-------
sdk/cwl/bin/{cwl-runner => arvados-cwl-runner} | 0
sdk/cwl/setup.py | 7 +-
sdk/go/keepclient/collectionreader.go | 2 +-
sdk/go/keepclient/collectionreader_test.go | 2 +-
sdk/go/keepclient/keepclient_test.go | 9 +-
sdk/go/keepclient/support.go | 32 ++++---
sdk/python/arvados/commands/run.py | 4 +-
services/api/lib/create_superuser_token.rb | 48 +++++++++++
services/api/script/create_superuser_token.rb | 30 ++-----
services/api/script/crunch-dispatch.rb | 2 +-
services/api/test/fixtures/nodes.yml | 2 +
.../api/test/unit/create_superuser_token_test.rb | 78 +++++++++++++++++
services/keep-web/anonymous.go | 2 +-
services/keep-web/doc.go | 13 ++-
services/keep-web/handler.go | 2 -
services/keepproxy/keepproxy.go | 21 +++--
22 files changed, 273 insertions(+), 95 deletions(-)
copy sdk/cwl/bin/{cwl-runner => arvados-cwl-runner} (100%)
create mode 100755 services/api/lib/create_superuser_token.rb
create mode 100644 services/api/test/unit/create_superuser_token_test.rb
via 767e1199d0e1bdf2b564b5c58a91d29141eb67d7 (commit)
via 78af1220d9e2ddf4d933d9a9487397414d8a3909 (commit)
via 8020be1d7230470b9052b11740bfed08e75029de (commit)
via 97128f31a4454621329ea12e031e88226a46d586 (commit)
via efb5501e40de0d49c285db0d7a22ff8a8593832c (commit)
via df25254be20988a99061cdaced053a17b0dc9bf9 (commit)
via 0dab1775b07d67ca25974c5b68189717b9f8952f (commit)
via 62817f027167fbe5fc5b66bbdc04261da16fe3ef (commit)
via 6207681aa301ad12d164aab24f52a210945af04b (commit)
via 4c2e89e152736ca977fc5c2fd1d4b58c4e87720f (commit)
via 7bf5f28cec1ff7be1e66925c3815c253989f9eb1 (commit)
via c271c93ccc5f6f187974d6a11c035652d6488401 (commit)
via 8dea5d629b93c1a8201cd347be19fc6f48239622 (commit)
via 4932c60225740aebd0105a2a0b360b3e5e049629 (commit)
via 2309c59e0d62a6c904e5600a139d429281e5a0fc (commit)
via 263cd68f5ae4b114d3c1c89f84be46b0f64f9c9e (commit)
via 8d3305862395e930a8a40383871426cfacebeab6 (commit)
via 2411624900867c1ca3884e84a6ad07e30b5d0802 (commit)
via a5e86befc352e31600399e9c4a476479d2155413 (commit)
via c830508697da4bf896be9ea7cea9c9cbfc5709b6 (commit)
via 8f237f88d58d9634814010e00ffe55e2cdde2334 (commit)
via a0b3480b644c79a44f3ad03030b573d40111fbaf (commit)
via 001e1ecfddb4f1d0cc26cdb415395ff47cde4914 (commit)
via c7137167b17cb07b2d2aca325fe028c54006741d (commit)
via e9153135c39388bf403ea94896f935ce80309b01 (commit)
via 07ad618c10f03f9d24970670f991791d4bd22b62 (commit)
from ce2bf31ba0831d9704f2edf82297f701b3b994df (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 767e1199d0e1bdf2b564b5c58a91d29141eb67d7
Merge: ce2bf31 78af122
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 3 23:20:27 2015 -0500
Merge branch '5824-keep-web' into 5824-keep-web-workbench
Conflicts:
services/keepproxy/keepproxy.go
diff --cc services/keepproxy/keepproxy.go
index bad0d22,79ed51e..2efc9ce
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@@ -89,17 -102,9 +102,9 @@@ func main()
defer os.Remove(pidfile)
}
- arv, err := arvadosclient.MakeArvadosClient()
- if err != nil {
- log.Fatalf("setting up arvados client: %v", err)
- }
- kc, err := keepclient.MakeKeepClient(&arv)
- if err != nil {
- log.Fatalf("setting up keep client: %v", err)
- }
kc.Want_replicas = default_replicas
-
kc.Client.Timeout = time.Duration(timeout) * time.Second
+ go RefreshServicesList(kc, 5*time.Minute, 3*time.Second)
listener, err = net.Listen("tcp", listen)
if err != nil {
commit 78af1220d9e2ddf4d933d9a9487397414d8a3909
Merge: 8020be1 efb5501
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 3 23:06:45 2015 -0500
5824: Merge branch 'master' into 5824-keep-web
commit 8020be1d7230470b9052b11740bfed08e75029de
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 3 22:50:50 2015 -0500
5824: Avoid sending empty slices through toRead chan. Fixes race in test case.
diff --git a/sdk/go/keepclient/collectionreader.go b/sdk/go/keepclient/collectionreader.go
index 68ecc6e..b532a16 100644
--- a/sdk/go/keepclient/collectionreader.go
+++ b/sdk/go/keepclient/collectionreader.go
@@ -210,7 +210,7 @@ GET:
close(r.errNotNil)
return
}
- for bOff, bLen := fs.Offset, dataSliceSize; bOff <= fs.Offset+fs.Len && bLen > 0; bOff += bLen {
+ for bOff, bLen := fs.Offset, dataSliceSize; bOff < fs.Offset+fs.Len && bLen > 0; bOff += bLen {
if bOff+bLen > fs.Offset+fs.Len {
bLen = fs.Offset + fs.Len - bOff
}
diff --git a/sdk/go/keepclient/collectionreader_test.go b/sdk/go/keepclient/collectionreader_test.go
index 9fb0d86..58a047c 100644
--- a/sdk/go/keepclient/collectionreader_test.go
+++ b/sdk/go/keepclient/collectionreader_test.go
@@ -203,7 +203,7 @@ func (s *CollectionReaderUnit) TestCollectionReaderCloseEarly(c *check.C) {
// doGet() should close toRead before sending any more bufs to it.
if what, ok := <-rdr.(*cfReader).toRead; ok {
- c.Errorf("Got %q, expected toRead to be closed", string(what))
+ c.Errorf("Got %q, expected toRead to be closed", what)
}
// Stub should have handled exactly one GET request.
commit 97128f31a4454621329ea12e031e88226a46d586
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 3 14:06:52 2015 -0500
5824: Turn off debug printfs unless enabled by calling program.
diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index df46386..4615ebc 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -143,10 +143,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
make(chan string)}
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, reader io.ReadCloser,
- writer io.WriteCloser, upload_status chan uploadStatus) {
+ func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer")
+ go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
writer.Write([]byte("foo"))
writer.Close()
@@ -178,7 +177,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
br1 := tr.MakeStreamReader()
- go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader")
+ go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0)
writer.Write([]byte("foo"))
writer.Close()
@@ -238,7 +237,7 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
func(kc *KeepClient, url string, reader io.ReadCloser,
writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer")
+ go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
writer.Write([]byte("foo"))
writer.Close()
diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 4a21024..8414afa 100644
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -9,12 +9,18 @@ import (
"io"
"io/ioutil"
"log"
+ "math/rand"
"net"
"net/http"
"strings"
"time"
)
+// Function used to emit debug messages. The easiest way to enable
+// keepclient debug messages in your application is to assign
+// log.Printf to DebugPrintf.
+var DebugPrintf = func(string, ...interface{}) {}
+
type keepService struct {
Uuid string `json:"uuid"`
Hostname string `json:"service_host"`
@@ -170,13 +176,13 @@ type uploadStatus struct {
}
func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.ReadCloser,
- upload_status chan<- uploadStatus, expectedLength int64, requestId string) {
+ upload_status chan<- uploadStatus, expectedLength int64, requestID int32) {
var req *http.Request
var err error
var url = fmt.Sprintf("%s/%s", host, hash)
if req, err = http.NewRequest("PUT", url, nil); err != nil {
- log.Printf("[%v] Error creating request PUT %v error: %v", requestId, url, err.Error())
+ log.Printf("[%08x] Error creating request PUT %v error: %v", requestID, url, err.Error())
upload_status <- uploadStatus{err, url, 0, 0, ""}
body.Close()
return
@@ -201,7 +207,7 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
var resp *http.Response
if resp, err = this.Client.Do(req); err != nil {
- log.Printf("[%v] Upload failed %v error: %v", requestId, url, err.Error())
+ log.Printf("[%08x] Upload failed %v error: %v", requestID, url, err.Error())
upload_status <- uploadStatus{err, url, 0, 0, ""}
return
}
@@ -217,13 +223,13 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
respbody, err2 := ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
response := strings.TrimSpace(string(respbody))
if err2 != nil && err2 != io.EOF {
- log.Printf("[%v] Upload %v error: %v response: %v", requestId, url, err2.Error(), response)
+ log.Printf("[%08x] Upload %v error: %v response: %v", requestID, url, err2.Error(), response)
upload_status <- uploadStatus{err2, url, resp.StatusCode, rep, response}
} else if resp.StatusCode == http.StatusOK {
- log.Printf("[%v] Upload %v success", requestId, url)
+ log.Printf("[%08x] Upload %v success", requestID, url)
upload_status <- uploadStatus{nil, url, resp.StatusCode, rep, response}
} else {
- log.Printf("[%v] Upload %v error: %v response: %v", requestId, url, resp.StatusCode, response)
+ log.Printf("[%08x] Upload %v error: %v response: %v", requestID, url, resp.StatusCode, response)
upload_status <- uploadStatus{errors.New(resp.Status), url, resp.StatusCode, rep, response}
}
}
@@ -233,9 +239,9 @@ func (this *KeepClient) putReplicas(
tr *streamer.AsyncStream,
expectedLength int64) (locator string, replicas int, err error) {
- // Take the hash of locator and timestamp in order to identify this
- // specific transaction in log statements.
- requestId := fmt.Sprintf("%x", md5.Sum([]byte(hash+time.Now().String())))[0:8]
+ // Generate an arbitrary ID to identify this specific
+ // transaction in debug logs.
+ requestID := rand.Int31()
// Calculate the ordering for uploading to servers
sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
@@ -280,8 +286,8 @@ func (this *KeepClient) putReplicas(
for active*replicasPerThread < remaining_replicas {
// Start some upload requests
if next_server < len(sv) {
- log.Printf("[%v] Begin upload %s to %s", requestId, hash, sv[next_server])
- go this.uploadToKeepServer(sv[next_server], hash, tr.MakeStreamReader(), upload_status, expectedLength, requestId)
+ log.Printf("[%08x] Begin upload %s to %s", requestID, hash, sv[next_server])
+ go this.uploadToKeepServer(sv[next_server], hash, tr.MakeStreamReader(), upload_status, expectedLength, requestID)
next_server += 1
active += 1
} else {
@@ -292,8 +298,8 @@ func (this *KeepClient) putReplicas(
}
}
}
- log.Printf("[%v] Replicas remaining to write: %v active uploads: %v",
- requestId, remaining_replicas, active)
+ log.Printf("[%08x] Replicas remaining to write: %v active uploads: %v",
+ requestID, remaining_replicas, active)
// Now wait for something to happen.
if active > 0 {
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 1a11896..79ed51e 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -84,6 +84,9 @@ func main() {
log.Fatalf("Error setting up arvados client %s", err.Error())
}
+ if os.Getenv("ARVADOS_DEBUG") != "" {
+ keepclient.DebugPrintf = log.Printf
+ }
kc, err := keepclient.MakeKeepClient(&arv)
if err != nil {
log.Fatalf("Error setting up keep client %s", err.Error())
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list