[ARVADOS] updated: c8055f0dd7c64c0dafcd89a74179e49fc1c14b87
Git user
git at public.curoverse.com
Wed May 11 11:46:44 EDT 2016
Summary of changes:
services/keepstore/azure_blob_volume.go | 94 +++++++++++++++++----------------
1 file changed, 49 insertions(+), 45 deletions(-)
via c8055f0dd7c64c0dafcd89a74179e49fc1c14b87 (commit)
via f6e53e280b26fa017036c6a0e14b1f4bcb7dad65 (commit)
from d5e4ac8e60999f53aaff84b58a1601e8315a9e64 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c8055f0dd7c64c0dafcd89a74179e49fc1c14b87
Author: radhika <radhika at curoverse.com>
Date: Wed May 11 11:43:42 2016 -0400
8556: Return metadata in checkTrashed so that the callers do not have to get blob metadata within that context again.
Remove trash marker in empty trash routince if we get not found error on a blob to avoid retrying it again and again.
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 40a9d31..6957a23 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -134,16 +134,16 @@ func (v *AzureBlobVolume) Check() error {
return nil
}
-// Return NotFoundError if trash marker is found on the block
-func (v *AzureBlobVolume) checkTrashed(loc string) (bool, error) {
+// Return NotFoundError if expires_at metadata attribute is found on the block
+func (v *AzureBlobVolume) checkTrashed(loc string) (bool, map[string]string, error) {
metadata, err := v.bsClient.GetBlobMetadata(v.containerName, loc)
if err != nil {
- return false, v.translateError(err)
+ return false, metadata, v.translateError(err)
}
if metadata["expires_at"] != "" {
- return true, nil
+ return true, metadata, nil
}
- return false, nil
+ return false, metadata, nil
}
// Get reads a Keep block that has been stored as a block blob in the
@@ -153,7 +153,7 @@ func (v *AzureBlobVolume) checkTrashed(loc string) (bool, error) {
// unexpectedly empty, assume a PutBlob operation is in progress, and
// wait for it to finish writing.
func (v *AzureBlobVolume) Get(loc string, buf []byte) (int, error) {
- trashed, err := v.checkTrashed(loc)
+ trashed, _, err := v.checkTrashed(loc)
if err != nil {
return 0, err
}
@@ -261,7 +261,7 @@ func (v *AzureBlobVolume) get(loc string, buf []byte) (int, error) {
// Compare the given data with existing stored data.
func (v *AzureBlobVolume) Compare(loc string, expect []byte) error {
- trashed, err := v.checkTrashed(loc)
+ trashed, _, err := v.checkTrashed(loc)
if err != nil {
return err
}
@@ -286,51 +286,41 @@ func (v *AzureBlobVolume) Put(loc string, block []byte) error {
return v.bsClient.CreateBlockBlobFromReader(v.containerName, loc, uint64(len(block)), bytes.NewReader(block), extraHeaders)
}
-func (v *AzureBlobVolume) addToMetadata(loc, name, value string) error {
- metadata, err := v.bsClient.GetBlobMetadata(v.containerName, loc)
- if err != nil {
- return err
- }
- metadata[name] = value
- return v.bsClient.SetBlobMetadata(v.containerName, loc, metadata)
-}
-
-func (v *AzureBlobVolume) removeFromMetadata(loc, name string) error {
- metadata, err := v.bsClient.GetBlobMetadata(v.containerName, loc)
- if err != nil {
- return err
- }
- metadata[name] = ""
- return v.bsClient.SetBlobMetadata(v.containerName, loc, metadata)
-}
-
// Touch updates the last-modified property of a block blob.
func (v *AzureBlobVolume) Touch(loc string) error {
if v.readonly {
return MethodDisabledError
}
- trashed, err := v.checkTrashed(loc)
+ trashed, metadata, err := v.checkTrashed(loc)
if err != nil {
return err
}
if trashed {
return os.ErrNotExist
}
- return v.addToMetadata(loc, "last_write_at", fmt.Sprintf("%d", time.Now()))
+
+ metadata["last_write_at"] = fmt.Sprintf("%d", time.Now())
+ return v.bsClient.SetBlobMetadata(v.containerName, loc, metadata)
}
-// Mtime returns the last-modified property of a block blob.
+// Mtime returns the last-write-at metadata property of a block blob.
+// For blobs stored before #8556, returns the last-modified property.
func (v *AzureBlobVolume) Mtime(loc string) (time.Time, error) {
- trashed, err := v.checkTrashed(loc)
+ trashed, metadata, err := v.checkTrashed(loc)
if err != nil {
return time.Time{}, err
}
if trashed {
return time.Time{}, os.ErrNotExist
}
- metadata, err := v.bsClient.GetBlobMetadata(v.containerName, loc)
- if err != nil {
- return time.Time{}, v.translateError(err)
+
+ // If it's an old blob before #8556, return last-modified property
+ if metadata["last_write_at"] == "" {
+ props, err := v.bsClient.GetBlobProperties(v.containerName, loc)
+ if err != nil {
+ return time.Time{}, err
+ }
+ return time.Parse(time.RFC1123, props.LastModified)
}
lastWriteAt, err := strconv.ParseInt(metadata["last_write_at"], 10, 64)
@@ -406,7 +396,12 @@ func (v *AzureBlobVolume) Trash(loc string) error {
})
}
// Mark as trash
- err = v.addToMetadata(loc, "expires_at", fmt.Sprintf("%d", time.Now().Add(trashLifetime).Unix()))
+ metadata, err := v.bsClient.GetBlobMetadata(v.containerName, loc)
+ if err != nil {
+ return err
+ }
+ metadata["expires_at"] = fmt.Sprintf("%d", time.Now().Add(trashLifetime).Unix())
+ err = v.bsClient.SetBlobMetadata(v.containerName, loc, metadata)
if err != nil {
return err
}
@@ -425,8 +420,10 @@ func (v *AzureBlobVolume) Untrash(loc string) error {
if metadata["expires_at"] == "" {
return os.ErrNotExist
}
+
// reset expires_at metadata attribute
- err = v.removeFromMetadata(loc, "expires_at")
+ metadata["expires_at"] = ""
+ err = v.bsClient.SetBlobMetadata(v.containerName, loc, metadata)
if err != nil {
return v.translateError(err)
}
@@ -519,6 +516,13 @@ blobListPage:
props, err := v.bsClient.GetBlobProperties(v.containerName, matches[2])
if err != nil {
log.Printf("EmptyTrash: GetBlobProperties(%v): %v", matches[2], err)
+ if os.IsNotExist(v.translateError(err)) {
+ // Delete the trash marker for non-existing blob
+ err = v.bsClient.DeleteBlob(v.containerName, b.Name, map[string]string{})
+ if err != nil {
+ log.Printf("EmptyTrash: Delete Trash Marker (%v): %v", b.Name, err)
+ }
+ }
continue
}
@@ -542,7 +546,7 @@ blobListPage:
// Delete the marker also
err = v.bsClient.DeleteBlob(v.containerName, b.Name, map[string]string{})
if err != nil {
- log.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
+ log.Printf("EmptyTrash: Delete Trash Marker(%v): %v", b.Name, err)
}
}
if resp.NextMarker == "" {
commit f6e53e280b26fa017036c6a0e14b1f4bcb7dad65
Author: radhika <radhika at curoverse.com>
Date: Wed May 11 10:39:46 2016 -0400
8556: checkTrashed returns true when trashed and no error.
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 7b144ca..40a9d31 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -141,7 +141,7 @@ func (v *AzureBlobVolume) checkTrashed(loc string) (bool, error) {
return false, v.translateError(err)
}
if metadata["expires_at"] != "" {
- return true, v.translateError(NotFoundError)
+ return true, nil
}
return false, nil
}
@@ -157,7 +157,7 @@ func (v *AzureBlobVolume) Get(loc string, buf []byte) (int, error) {
if err != nil {
return 0, err
}
- if trashed == true {
+ if trashed {
return 0, os.ErrNotExist
}
var deadline time.Time
@@ -265,7 +265,7 @@ func (v *AzureBlobVolume) Compare(loc string, expect []byte) error {
if err != nil {
return err
}
- if trashed == true {
+ if trashed {
return os.ErrNotExist
}
rdr, err := v.bsClient.GetBlob(v.containerName, loc)
@@ -313,7 +313,7 @@ func (v *AzureBlobVolume) Touch(loc string) error {
if err != nil {
return err
}
- if trashed == true {
+ if trashed {
return os.ErrNotExist
}
return v.addToMetadata(loc, "last_write_at", fmt.Sprintf("%d", time.Now()))
@@ -325,7 +325,7 @@ func (v *AzureBlobVolume) Mtime(loc string) (time.Time, error) {
if err != nil {
return time.Time{}, err
}
- if trashed == true {
+ if trashed {
return time.Time{}, os.ErrNotExist
}
metadata, err := v.bsClient.GetBlobMetadata(v.containerName, loc)
@@ -335,7 +335,7 @@ func (v *AzureBlobVolume) Mtime(loc string) (time.Time, error) {
lastWriteAt, err := strconv.ParseInt(metadata["last_write_at"], 10, 64)
if err != nil {
- return time.Time{}, v.translateError(err)
+ return time.Time{}, err
}
return time.Unix(lastWriteAt, 0), nil
}
@@ -423,7 +423,7 @@ func (v *AzureBlobVolume) Untrash(loc string) error {
return v.translateError(err)
}
if metadata["expires_at"] == "" {
- return v.translateError(NotFoundError)
+ return os.ErrNotExist
}
// reset expires_at metadata attribute
err = v.removeFromMetadata(loc, "expires_at")
@@ -508,7 +508,7 @@ blobListPage:
blocksInTrash++
deadline, err := strconv.ParseInt(matches[1], 10, 64)
if err != nil {
- log.Printf("EmptyTrash: %v: ParseInt(%v): %v", matches[1], err)
+ log.Printf("EmptyTrash: ParseInt(%v): %v", matches[1], err)
continue
}
if deadline > time.Now().Unix() {
@@ -518,14 +518,14 @@ blobListPage:
// Get the Etag before checking expires_at, and use it to delete blob
props, err := v.bsClient.GetBlobProperties(v.containerName, matches[2])
if err != nil {
- log.Printf("EmptyTrash: %v: GetBlobProperties(%v): %v", matches[2], err)
+ log.Printf("EmptyTrash: GetBlobProperties(%v): %v", matches[2], err)
continue
}
// Make sure the marker is for the current block, not an older one
metadata, err := v.bsClient.GetBlobMetadata(v.containerName, matches[2])
if err != nil {
- log.Printf("EmptyTrash: %v: GetBlobMetadata(%v): %v", matches[2], err)
+ log.Printf("EmptyTrash: GetBlobMetadata(%v): %v", matches[2], err)
continue
}
if metadata["expires_at"] == matches[1] {
@@ -533,7 +533,7 @@ blobListPage:
"If-Match": props.Etag,
})
if err != nil {
- log.Printf("EmptyTrash: %v: DeleteBlob(%v): %v", matches[2], err)
+ log.Printf("EmptyTrash: DeleteBlob(%v): %v", matches[2], err)
continue
}
blocksDeleted++
@@ -542,7 +542,7 @@ blobListPage:
// Delete the marker also
err = v.bsClient.DeleteBlob(v.containerName, b.Name, map[string]string{})
if err != nil {
- log.Printf("EmptyTrash: %v: DeleteBlob(%v): %v", b.Name, err)
+ log.Printf("EmptyTrash: DeleteBlob(%v): %v", b.Name, err)
}
}
if resp.NextMarker == "" {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list