[ARVADOS] updated: fed6d3b24a6c49df01601dfdc27182fd0770b013
git at public.curoverse.com
git at public.curoverse.com
Fri Mar 20 23:00:34 EDT 2015
Summary of changes:
sdk/go/arvadosclient/arvadosclient.go | 28 +++++++++++-
sdk/go/arvadosclient/arvadosclient_test.go | 16 ++++++-
...tion_portable_data_hash_with_hinted_manifest.rb | 36 +++++++++------
services/keepstore/keepstore.go | 7 ++-
services/keepstore/trash_worker.go | 15 +++++--
services/keepstore/trash_worker_test.go | 51 +++++++++++++++++++---
6 files changed, 126 insertions(+), 27 deletions(-)
via fed6d3b24a6c49df01601dfdc27182fd0770b013 (commit)
via 6c14ca8fe128f4db3bafe4edd1cc5551bc85e1ef (commit)
via a307f4b97a91ae47e3498dac36d0a47b1c0eedcd (commit)
via 2ca0934c1daf79d5c1b959c8b3d0f36b5dd57bc4 (commit)
via 7e5416c8d769fa6520dad72cd0612f1776d88219 (commit)
via 1d0aef8fb7bdad6d28976255919f58b719a4342b (commit)
from ac037435f2ee9688957f1c0be6bfa9066ac027ba (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fed6d3b24a6c49df01601dfdc27182fd0770b013
Merge: 6c14ca8 a307f4b
Author: Radhika Chippada <radhika at curoverse.com>
Date: Fri Mar 20 22:59:42 2015 -0400
Merge branch 'master' into 3762-delete-list-worker
commit 6c14ca8fe128f4db3bafe4edd1cc5551bc85e1ef
Author: Radhika Chippada <radhika at curoverse.com>
Date: Fri Mar 20 22:58:05 2015 -0400
3762: Expose Discovery func in arvadosclient. Use Discovery to get defaultTrashLifetime in trash worker.
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 7c24426..f606a68 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -22,7 +22,7 @@ var MissingArvadosApiToken = errors.New("Missing required environment variable A
type ArvadosApiError struct {
error
HttpStatusCode int
- HttpStatus string
+ HttpStatus string
}
func (e ArvadosApiError) Error() string { return e.error.Error() }
@@ -92,7 +92,9 @@ func (this ArvadosClient) CallRaw(method string, resource string, uuid string, a
Scheme: "https",
Host: this.ApiServer}
- u.Path = "/arvados/v1"
+ if resource != API_DISCOVERY_RESOURCE {
+ u.Path = "/arvados/v1"
+ }
if resource != "" {
u.Path = u.Path + "/" + resource
@@ -247,3 +249,25 @@ func (this ArvadosClient) Update(resource string, uuid string, parameters Dict,
func (this ArvadosClient) List(resource string, parameters Dict, output interface{}) (err error) {
return this.Call("GET", resource, "", "", parameters, output)
}
+
+// API Discovery
+//
+// parameter - name of parameter to be discovered
+// return
+// valueMap - Dict key value pair of the discovered parameter
+// err - error accessing the resource, or nil if no error
+var API_DISCOVERY_RESOURCE string = "discovery/v1/apis/arvados/v1/rest"
+
+var DISCOVERY Dict
+
+func (this ArvadosClient) Discovery(parameter string) (valueMap Dict, err error) {
+ if len(DISCOVERY) == 0 {
+ DISCOVERY = make(Dict)
+ this.Call("GET", API_DISCOVERY_RESOURCE, "", "", nil, &DISCOVERY)
+ }
+
+ valueMap = make(Dict)
+ valueMap[parameter] = DISCOVERY[parameter]
+
+ return valueMap, err
+}
diff --git a/sdk/go/arvadosclient/arvadosclient_test.go b/sdk/go/arvadosclient/arvadosclient_test.go
index 1af964d..97c9b18 100644
--- a/sdk/go/arvadosclient/arvadosclient_test.go
+++ b/sdk/go/arvadosclient/arvadosclient_test.go
@@ -1,8 +1,8 @@
package arvadosclient
import (
- . "gopkg.in/check.v1"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ . "gopkg.in/check.v1"
"net/http"
"os"
"testing"
@@ -100,3 +100,17 @@ func (s *ServerRequiredSuite) TestErrorResponse(c *C) {
c.Assert(err.(ArvadosApiError).HttpStatusCode, Equals, 404)
}
}
+
+func (s *ServerRequiredSuite) TestAPIDiscovery_Get_defaultTrashLifetime(c *C) {
+ arv, err := MakeArvadosClient()
+ valueMap, err := arv.Discovery("defaultTrashLifetime")
+ c.Assert(err, IsNil)
+ c.Assert(valueMap["defaultTrashLifetime"], NotNil)
+}
+
+func (s *ServerRequiredSuite) TestAPIDiscovery_Get_noSuchParameter(c *C) {
+ arv, err := MakeArvadosClient()
+ valueMap, err := arv.Discovery("noSuchParameter")
+ c.Assert(err, IsNil)
+ c.Assert(valueMap["noSuchParameter"], IsNil)
+}
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index a363bac..e7ac203 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -4,6 +4,7 @@ import (
"bytes"
"flag"
"fmt"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"io/ioutil"
"log"
@@ -288,8 +289,12 @@ func main() {
go RunPullWorker(pullq, keepClient)
// Initialize the trashq and worker
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ log.Fatalf("Error setting up arvados client %s", err.Error())
+ }
trashq = NewWorkQueue()
- go RunTrashWorker(trashq)
+ go RunTrashWorker(&arv, trashq)
// Shut down the server gracefully (by closing the listener)
// if SIGTERM is received.
diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go
index 6b8d60b..bc99e96 100644
--- a/services/keepstore/trash_worker.go
+++ b/services/keepstore/trash_worker.go
@@ -1,6 +1,7 @@
package main
import (
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"log"
"time"
)
@@ -13,9 +14,17 @@ import (
Repeat
*/
-var DEFAULT_TRASH_LIFE_TIME int64 = 1209600 // Use 2 weeks for now
+var defaultTrashLifetime int64 = 0
+
+func RunTrashWorker(arv *arvadosclient.ArvadosClient, trashq *WorkQueue) {
+ if arv != nil {
+ defaultTrashLifetimeMap, err := arv.Discovery("defaultTrashLifetime")
+ if err != nil {
+ log.Fatalf("Error setting up arvados client %s", err.Error())
+ }
+ defaultTrashLifetime = int64(defaultTrashLifetimeMap["defaultTrashLifetime"].(float64))
+ }
-func RunTrashWorker(trashq *WorkQueue) {
nextItem := trashq.NextItem
for item := range nextItem {
trashRequest := item.(TrashRequest)
@@ -36,7 +45,7 @@ func TrashItem(trashRequest TrashRequest) (err error) {
if err == nil {
if trashRequest.BlockMtime == mtime.Unix() {
currentTime := time.Now().Unix()
- if (currentTime - trashRequest.BlockMtime) > DEFAULT_TRASH_LIFE_TIME {
+ if (currentTime - trashRequest.BlockMtime) > defaultTrashLifetime {
err = volume.Delete(trashRequest.Locator)
}
}
diff --git a/services/keepstore/trash_worker_test.go b/services/keepstore/trash_worker_test.go
index e4f7fb3..dee8f17 100644
--- a/services/keepstore/trash_worker_test.go
+++ b/services/keepstore/trash_worker_test.go
@@ -2,6 +2,8 @@ package main
import (
"container/list"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "log"
"testing"
"time"
)
@@ -19,6 +21,8 @@ type TrashWorkerTestData struct {
CreateInVolume1 bool
UseDelayToCreate bool
+ UseDefaultTrashTime bool
+
DeleteLocator string
ExpectLocator1 bool
@@ -154,20 +158,41 @@ func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
performTrashWorkerTest(testData, t)
}
+/* Allow defaultTrashLifetime to be used. Thus, the newly created block
+ will not be deleted becuase its Mtime is within the trash life time.
+*/
+func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(t *testing.T) {
+ testData := TrashWorkerTestData{
+ Locator1: TEST_HASH,
+ Block1: TEST_BLOCK,
+
+ Locator2: TEST_HASH_2,
+ Block2: TEST_BLOCK_2,
+
+ CreateData: true,
+ CreateInVolume1: true,
+
+ UseDefaultTrashTime: true,
+
+ DeleteLocator: TEST_HASH, // locator 1
+
+ // Since defaultTrashLifetime is in effect, block won't be deleted.
+ ExpectLocator1: true,
+ ExpectLocator2: true,
+ }
+ performTrashWorkerTest(testData, t)
+}
+
/* Perform the test */
func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
// Create Keep Volumes
KeepVM = MakeTestVolumeManager(2)
- // Set trash life time delta to 0 so that the test can delete the blocks right after create
- DEFAULT_TRASH_LIFE_TIME = 0
-
// Delete from volume will not take place if the block MTime is within permission_ttl
permission_ttl = time.Duration(1) * time.Second
- vols := KeepVM.Volumes()
-
// Put test content
+ vols := KeepVM.Volumes()
if testData.CreateData {
vols[0].Put(testData.Locator1, testData.Block1)
vols[0].Put(testData.Locator1+".meta", []byte("metadata"))
@@ -199,9 +224,21 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
trashList := list.New()
trashList.PushBack(trashRequest)
trashq = NewWorkQueue()
- go RunTrashWorker(trashq)
+
+ // Trash worker would not delete block if its Mtime is within defaultTrashLifetime
+ // Hence, we will have to bypass it to allow the deletion to succeed.
+ if !testData.UseDefaultTrashTime {
+ go RunTrashWorker(nil, trashq)
+ } else {
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ log.Fatalf("Error setting up arvados client %s", err.Error())
+ }
+ go RunTrashWorker(&arv, trashq)
+ }
+
trashq.ReplaceQueue(trashList)
- time.Sleep(10 * time.Millisecond) // give it a moment to finish processing the trash list
+ time.Sleep(10 * time.Millisecond) // give a moment to finish processing the list
// Verify Locator1 to be un/deleted as expected
data, _ := GetBlock(testData.Locator1, false)
commit 2ca0934c1daf79d5c1b959c8b3d0f36b5dd57bc4
Merge: ac03743 7e5416c
Author: Radhika Chippada <radhika at curoverse.com>
Date: Fri Mar 20 15:56:38 2015 -0400
Merge branch 'master' into 3762-delete-list-worker
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list