[ARVADOS] created: 1.2.0-353-g3b65c09dc
Git user
git at public.curoverse.com
Mon Nov 12 10:41:52 EST 2018
at 3b65c09dc878abf22a04f224ac8496380b307f7c (commit)
commit 3b65c09dc878abf22a04f224ac8496380b307f7c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Nov 12 10:40:27 2018 -0500
14397: Use {sha1}-dev as version in dev builds of Go programs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 9674ad5d4..2fb82554a 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -689,7 +689,7 @@ do_test_once() {
# before trying "go test". Otherwise, coverage-reporting
# mode makes Go show the wrong line numbers when reporting
# compilation errors.
- go get -t "git.curoverse.com/arvados.git/$1" && \
+ go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git head)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
[[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
if [[ -n "${testargs[$1]}" ]]
@@ -757,7 +757,7 @@ do_install_once() {
timer_reset
if [[ "$2" == "go" ]]
then
- go get -t "git.curoverse.com/arvados.git/$1"
+ go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git head)-dev}" -t "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
then
# $3 can name a path directory for us to use, including trailing
commit 1dfede4c84c961230954a4dad9ab348029dfa128
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Nov 12 09:27:12 2018 -0500
14397: Fix unchecked errors in S3 block listing.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 02d9dcf91..68406d5a2 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -509,8 +509,7 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
PageSize: v.IndexPageSize,
Stats: &v.bucket.stats,
}
- for data, recent := dataL.First(), recentL.First(); data != nil; data = dataL.Next() {
- v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.ListOps)
+ for data, recent := dataL.First(), recentL.First(); data != nil && dataL.Error() == nil; data = dataL.Next() {
if data.Key >= "g" {
// Conveniently, "recent/*" and "trash/*" are
// lexically greater than all hex-encoded data
@@ -529,7 +528,7 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
stamp := data
// Advance to the corresponding recent/X marker, if any
- for recent != nil {
+ for recent != nil && recentL.Error() == nil {
if cmp := strings.Compare(recent.Key[7:], data.Key); cmp < 0 {
recent = recentL.Next()
continue
@@ -544,13 +543,16 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
break
}
}
+ if err := recentL.Error(); err != nil {
+ return err
+ }
t, err := time.Parse(time.RFC3339, stamp.LastModified)
if err != nil {
return err
}
fmt.Fprintf(writer, "%s+%d %d\n", data.Key, data.Size, t.UnixNano())
}
- return nil
+ return dataL.Error()
}
// Trash a Keep block.
commit 472d5d0daa87835ab04c02a6298fdc338d1a0446
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Nov 12 09:24:20 2018 -0500
14397: Fix S3 ListOps stats.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index ae1342512..02d9dcf91 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -501,14 +501,14 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
Bucket: v.bucket.Bucket,
Prefix: prefix,
PageSize: v.IndexPageSize,
+ Stats: &v.bucket.stats,
}
recentL := s3Lister{
Bucket: v.bucket.Bucket,
Prefix: "recent/" + prefix,
PageSize: v.IndexPageSize,
+ Stats: &v.bucket.stats,
}
- v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.ListOps)
- v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.ListOps)
for data, recent := dataL.First(), recentL.First(); data != nil; data = dataL.Next() {
v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.ListOps)
if data.Key >= "g" {
@@ -532,12 +532,10 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
for recent != nil {
if cmp := strings.Compare(recent.Key[7:], data.Key); cmp < 0 {
recent = recentL.Next()
- v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.ListOps)
continue
} else if cmp == 0 {
stamp = recent
recent = recentL.Next()
- v.bucket.stats.Tick(&v.bucket.stats.Ops, &v.bucket.stats.ListOps)
break
} else {
// recent/X marker is missing: we'll
@@ -873,6 +871,7 @@ func (v *S3Volume) EmptyTrash() {
Bucket: v.bucket.Bucket,
Prefix: "trash/",
PageSize: v.IndexPageSize,
+ Stats: &v.bucket.stats,
}
for trash := trashL.First(); trash != nil; trash = trashL.Next() {
todo <- trash
@@ -890,6 +889,7 @@ type s3Lister struct {
Bucket *s3.Bucket
Prefix string
PageSize int
+ Stats *s3bucketStats
nextMarker string
buf []s3.Key
err error
@@ -918,6 +918,7 @@ func (lister *s3Lister) Error() error {
}
func (lister *s3Lister) getPage() {
+ lister.Stats.Tick(&lister.Stats.Ops, &lister.Stats.ListOps)
resp, err := lister.Bucket.List(lister.Prefix, "", lister.nextMarker, lister.PageSize)
lister.nextMarker = ""
if err != nil {
commit e6dcabea4865284e34efc50eda6e92b062b7adda
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Nov 9 15:02:54 2018 -0500
14397: Improve error reporting.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index f78084dbc..ae1342512 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -624,8 +624,10 @@ func (v *S3Volume) safeCopy(dst, src string) error {
MetadataDirective: "REPLACE",
}, v.bucket.Name+"/"+src)
err = v.translateError(err)
- if err != nil {
+ if os.IsNotExist(err) {
return err
+ } else if err != nil {
+ return fmt.Errorf("PutCopy(%q ← %q): %s", dst, v.bucket.Name+"/"+src, err)
}
if t, err := time.Parse(time.RFC3339Nano, resp.LastModified); err != nil {
return fmt.Errorf("PutCopy succeeded but did not return a timestamp: %q: %s", resp.LastModified, err)
diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go
index cbb831ebc..8a9fedfb7 100644
--- a/services/keepstore/trash_worker.go
+++ b/services/keepstore/trash_worker.go
@@ -51,24 +51,24 @@ func TrashItem(trashRequest TrashRequest) {
for _, volume := range volumes {
mtime, err := volume.Mtime(trashRequest.Locator)
if err != nil {
- log.Printf("%v Delete(%v): %v", volume, trashRequest.Locator, err)
+ log.Printf("%v Trash(%v): %v", volume, trashRequest.Locator, err)
continue
}
if trashRequest.BlockMtime != mtime.UnixNano() {
- log.Printf("%v Delete(%v): stored mtime %v does not match trash list value %v", volume, trashRequest.Locator, mtime.UnixNano(), trashRequest.BlockMtime)
+ log.Printf("%v Trash(%v): stored mtime %v does not match trash list value %v", volume, trashRequest.Locator, mtime.UnixNano(), trashRequest.BlockMtime)
continue
}
if !theConfig.EnableDelete {
- err = errors.New("did not delete block because EnableDelete is false")
+ err = errors.New("skipping because EnableDelete is false")
} else {
err = volume.Trash(trashRequest.Locator)
}
if err != nil {
- log.Printf("%v Delete(%v): %v", volume, trashRequest.Locator, err)
+ log.Printf("%v Trash(%v): %v", volume, trashRequest.Locator, err)
} else {
- log.Printf("%v Delete(%v) OK", volume, trashRequest.Locator)
+ log.Printf("%v Trash(%v) OK", volume, trashRequest.Locator)
}
}
}
commit 1fb513b24cb808356939083a3df99bce9e702f77
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Nov 9 14:53:08 2018 -0500
14397: Update goamz to fix v4 signature.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/vendor/vendor.json b/vendor/vendor.json
index a6b4c17cf..2bd409b8a 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -3,24 +3,24 @@
"ignore": "test",
"package": [
{
- "checksumSHA1": "jf7K+UTQNIzRdlG5F4zX/8b++/E=",
+ "checksumSHA1": "f3QOPH0L/cCtLRzi7PI838GujAQ=",
"origin": "github.com/curoverse/goamz/aws",
"path": "github.com/AdRoll/goamz/aws",
- "revision": "21e563311c2dc5ac53464a2c31cb91fb833c6cb9",
+ "revision": "d0a83495df264858eda1fe89ebfac66b22a68ba3",
"revisionTime": "2017-07-27T13:52:37Z"
},
{
- "checksumSHA1": "9nUwQXI+pNxZo6bnR7NslpMpfPI=",
+ "checksumSHA1": "+T3Mbw98mQoOgX+z/UpumfTHFlU=",
"origin": "github.com/curoverse/goamz/s3",
"path": "github.com/AdRoll/goamz/s3",
- "revision": "21e563311c2dc5ac53464a2c31cb91fb833c6cb9",
+ "revision": "d0a83495df264858eda1fe89ebfac66b22a68ba3",
"revisionTime": "2017-07-27T13:52:37Z"
},
{
"checksumSHA1": "tvxbsTkdjB0C/uxEglqD6JfVnMg=",
"origin": "github.com/curoverse/goamz/s3/s3test",
"path": "github.com/AdRoll/goamz/s3/s3test",
- "revision": "21e563311c2dc5ac53464a2c31cb91fb833c6cb9",
+ "revision": "d0a83495df264858eda1fe89ebfac66b22a68ba3",
"revisionTime": "2017-07-27T13:52:37Z"
},
{
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list