[ARVADOS] updated: 7f28788a82182d96c9bf0792fe6c1682714ea28f
git at public.curoverse.com
git at public.curoverse.com
Mon Mar 23 12:02:17 EDT 2015
Summary of changes:
.../app/assets/javascripts/job_log_graph.js | 76 ++++++++++++-------
apps/workbench/app/helpers/collections_helper.rb | 17 +++++
.../app/views/application/_show_sharing.html.erb | 6 ++
.../app/views/collections/_show_files.html.erb | 3 +-
apps/workbench/config/application.default.yml | 7 +-
apps/workbench/config/initializers/mime_types.rb | 15 ++++
.../controllers/application_controller_test.rb | 4 +-
.../controllers/collections_controller_test.rb | 29 +++++++-
.../test/helpers/collections_helper_test.rb | 34 +++++++++
apps/workbench/test/integration/websockets_test.rb | 87 ++++++++++++----------
sdk/go/arvadosclient/arvadosclient.go | 34 ++-------
sdk/go/arvadosclient/arvadosclient_test.go | 16 +---
sdk/go/keepclient/keepclient.go | 7 +-
services/api/lib/simulate_job_log.rb | 10 ++-
services/api/test/fixtures/collections.yml | 24 ++++++
services/api/test/fixtures/logs.yml | 29 ++++++--
.../api_client_authorizations_scopes_test.rb | 2 +-
services/api/test/integration/websocket_test.rb | 8 +-
services/api/test/unit/log_test.rb | 45 +++++++----
services/keepstore/keepstore.go | 7 +-
services/keepstore/trash_worker.go | 15 +---
services/keepstore/trash_worker_test.go | 29 +++-----
22 files changed, 322 insertions(+), 182 deletions(-)
create mode 100644 apps/workbench/test/helpers/collections_helper_test.rb
via 7f28788a82182d96c9bf0792fe6c1682714ea28f (commit)
via b01138f2ed3b666fd6075a6d77caddb5d5ca91e1 (commit)
via 23721458d7ec8c44c4b54f45eb7d2a4e6d7cdce1 (commit)
via d8326579d327530adcf4118b48947d3812845c33 (commit)
via 6381b9e85f278b0c8cb45ffccd89ca1b1bc4d3ee (commit)
via df9955d76e3103e747360d2c27914d79b97fdf6f (commit)
via 5cb7bacb7326d1aae1b5b323614d6b46d93d0e99 (commit)
via a51029fa0b4466ac4f5dff204f5e1977753a88ad (commit)
via 3c3d04e7e6ec2b21fed84faa2789c4bac3fd4355 (commit)
via 10d3f91d4b5619172dd5d8fbf3e8aa863b673e2f (commit)
via 1d20cb1d8ff6c4ba678fb5387265f99b4912a028 (commit)
via 2fdfdc0a6c8fe550f225cbc16e1582406050cd83 (commit)
via cb152be857f9fe25e039202af217656c0ca444d2 (commit)
via d9561b0197cba4a9a0f70929eb03c892df5c4543 (commit)
via 4a8dc955e10f0d27afa6a704843d49326bd8d3b5 (commit)
via 2606f4df5861ff34ca9e42389a1509e77c102c5a (commit)
via 03e8ba641e0161f15d96351e2432805977fd9d57 (commit)
via a6ea50199a4aef32094bb6c4e91aa702c02aaca0 (commit)
via ab9955f1ab1aef27b3eecf5d739bf0934d29cdfd (commit)
via b4e040f6864ba10b19009e21744119194a86567f (commit)
via 52f124ea43c941d8500a365945fb9bd9b4b99177 (commit)
via 4e72d3799b87374412e01805bf8b27a4620cd1db (commit)
via a9c8f2dd15af7843ac4d86e5820ab0771cc5fd78 (commit)
via f75e2ecbcb1e511e26ae34fb4ebb9bf81e1cbe8f (commit)
via c3604c1cc57556520634f9af57213abc80c86da9 (commit)
via f46eed2ad6bc456cd57ed363e1adc8d60f802f39 (commit)
via 3c191493c9c4d778525751d15dea8e83ed61fda6 (commit)
via fe6bad60cace3d6e0f611b1c07fc7f4cb593c48e (commit)
via 12540bb0c62e2a498a70f64198055f1d48deeb17 (commit)
from fed6d3b24a6c49df01601dfdc27182fd0770b013 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 7f28788a82182d96c9bf0792fe6c1682714ea28f
Author: Radhika Chippada <radhika at curoverse.com>
Date: Mon Mar 23 12:01:05 2015 -0400
3762: undo the api server discovery document update. trash worker can instead use permission_ttl in place of defaultTrashLifetime because these two will be the same.
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 2cdfe75..4c16398 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -23,7 +23,7 @@ var MissingArvadosApiToken = errors.New("Missing required environment variable A
type ArvadosApiError struct {
error
HttpStatusCode int
- HttpStatus string
+ HttpStatus string
}
func (e ArvadosApiError) Error() string { return e.error.Error() }
@@ -94,9 +94,7 @@ func (this ArvadosClient) CallRaw(method string, resource string, uuid string, a
Scheme: "https",
Host: this.ApiServer}
- if resource != API_DISCOVERY_RESOURCE {
- u.Path = "/arvados/v1"
- }
+ u.Path = "/arvados/v1"
if resource != "" {
u.Path = u.Path + "/" + resource
@@ -251,25 +249,3 @@ func (this ArvadosClient) Update(resource string, uuid string, parameters Dict,
func (this ArvadosClient) List(resource string, parameters Dict, output interface{}) (err error) {
return this.Call("GET", resource, "", "", parameters, output)
}
-
-// API Discovery
-//
-// parameter - name of parameter to be discovered
-// return
-// valueMap - Dict key value pair of the discovered parameter
-// err - error accessing the resource, or nil if no error
-var API_DISCOVERY_RESOURCE string = "discovery/v1/apis/arvados/v1/rest"
-
-var DISCOVERY Dict
-
-func (this ArvadosClient) Discovery(parameter string) (valueMap Dict, err error) {
- if len(DISCOVERY) == 0 {
- DISCOVERY = make(Dict)
- this.Call("GET", API_DISCOVERY_RESOURCE, "", "", nil, &DISCOVERY)
- }
-
- valueMap = make(Dict)
- valueMap[parameter] = DISCOVERY[parameter]
-
- return valueMap, err
-}
diff --git a/sdk/go/arvadosclient/arvadosclient_test.go b/sdk/go/arvadosclient/arvadosclient_test.go
index 97c9b18..1af964d 100644
--- a/sdk/go/arvadosclient/arvadosclient_test.go
+++ b/sdk/go/arvadosclient/arvadosclient_test.go
@@ -1,8 +1,8 @@
package arvadosclient
import (
- "git.curoverse.com/arvados.git/sdk/go/arvadostest"
. "gopkg.in/check.v1"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
"net/http"
"os"
"testing"
@@ -100,17 +100,3 @@ func (s *ServerRequiredSuite) TestErrorResponse(c *C) {
c.Assert(err.(ArvadosApiError).HttpStatusCode, Equals, 404)
}
}
-
-func (s *ServerRequiredSuite) TestAPIDiscovery_Get_defaultTrashLifetime(c *C) {
- arv, err := MakeArvadosClient()
- valueMap, err := arv.Discovery("defaultTrashLifetime")
- c.Assert(err, IsNil)
- c.Assert(valueMap["defaultTrashLifetime"], NotNil)
-}
-
-func (s *ServerRequiredSuite) TestAPIDiscovery_Get_noSuchParameter(c *C) {
- arv, err := MakeArvadosClient()
- valueMap, err := arv.Discovery("noSuchParameter")
- c.Assert(err, IsNil)
- c.Assert(valueMap["noSuchParameter"], IsNil)
-}
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index e7ac203..a363bac 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -4,7 +4,6 @@ import (
"bytes"
"flag"
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"io/ioutil"
"log"
@@ -289,12 +288,8 @@ func main() {
go RunPullWorker(pullq, keepClient)
// Initialize the trashq and worker
- arv, err := arvadosclient.MakeArvadosClient()
- if err != nil {
- log.Fatalf("Error setting up arvados client %s", err.Error())
- }
trashq = NewWorkQueue()
- go RunTrashWorker(&arv, trashq)
+ go RunTrashWorker(trashq)
// Shut down the server gracefully (by closing the listener)
// if SIGTERM is received.
diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go
index bc99e96..72b9a46 100644
--- a/services/keepstore/trash_worker.go
+++ b/services/keepstore/trash_worker.go
@@ -1,7 +1,6 @@
package main
import (
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"log"
"time"
)
@@ -14,17 +13,7 @@ import (
Repeat
*/
-var defaultTrashLifetime int64 = 0
-
-func RunTrashWorker(arv *arvadosclient.ArvadosClient, trashq *WorkQueue) {
- if arv != nil {
- defaultTrashLifetimeMap, err := arv.Discovery("defaultTrashLifetime")
- if err != nil {
- log.Fatalf("Error setting up arvados client %s", err.Error())
- }
- defaultTrashLifetime = int64(defaultTrashLifetimeMap["defaultTrashLifetime"].(float64))
- }
-
+func RunTrashWorker(trashq *WorkQueue) {
nextItem := trashq.NextItem
for item := range nextItem {
trashRequest := item.(TrashRequest)
@@ -45,7 +34,7 @@ func TrashItem(trashRequest TrashRequest) (err error) {
if err == nil {
if trashRequest.BlockMtime == mtime.Unix() {
currentTime := time.Now().Unix()
- if (currentTime - trashRequest.BlockMtime) > defaultTrashLifetime {
+ if time.Duration(currentTime-trashRequest.BlockMtime)*time.Second >= permission_ttl {
err = volume.Delete(trashRequest.Locator)
}
}
diff --git a/services/keepstore/trash_worker_test.go b/services/keepstore/trash_worker_test.go
index dee8f17..3031c25 100644
--- a/services/keepstore/trash_worker_test.go
+++ b/services/keepstore/trash_worker_test.go
@@ -2,8 +2,6 @@ package main
import (
"container/list"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "log"
"testing"
"time"
)
@@ -21,7 +19,7 @@ type TrashWorkerTestData struct {
CreateInVolume1 bool
UseDelayToCreate bool
- UseDefaultTrashTime bool
+ UseTrashLifeTime bool
DeleteLocator string
@@ -158,7 +156,7 @@ func TestTrashWorkerIntegration_TwoDifferentLocatorsInVolume1(t *testing.T) {
performTrashWorkerTest(testData, t)
}
-/* Allow defaultTrashLifetime to be used. Thus, the newly created block
+/* Allow default Trash Life time to be used. Thus, the newly created block
will not be deleted becuase its Mtime is within the trash life time.
*/
func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(t *testing.T) {
@@ -172,11 +170,11 @@ func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(
CreateData: true,
CreateInVolume1: true,
- UseDefaultTrashTime: true,
+ UseTrashLifeTime: true,
DeleteLocator: TEST_HASH, // locator 1
- // Since defaultTrashLifetime is in effect, block won't be deleted.
+ // Since trash life time is in effect, block won't be deleted.
ExpectLocator1: true,
ExpectLocator2: true,
}
@@ -185,12 +183,11 @@ func TestTrashWorkerIntegration_SameLocatorInTwoVolumesWithDefaultTrashLifeTime(
/* Perform the test */
func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
+ actual_permission_ttl := permission_ttl
+
// Create Keep Volumes
KeepVM = MakeTestVolumeManager(2)
- // Delete from volume will not take place if the block MTime is within permission_ttl
- permission_ttl = time.Duration(1) * time.Second
-
// Put test content
vols := KeepVM.Volumes()
if testData.CreateData {
@@ -225,17 +222,12 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
trashList.PushBack(trashRequest)
trashq = NewWorkQueue()
- // Trash worker would not delete block if its Mtime is within defaultTrashLifetime
+ // Trash worker would not delete block if its Mtime is within trash life time.
// Hence, we will have to bypass it to allow the deletion to succeed.
- if !testData.UseDefaultTrashTime {
- go RunTrashWorker(nil, trashq)
- } else {
- arv, err := arvadosclient.MakeArvadosClient()
- if err != nil {
- log.Fatalf("Error setting up arvados client %s", err.Error())
- }
- go RunTrashWorker(&arv, trashq)
+ if !testData.UseTrashLifeTime {
+ permission_ttl = time.Duration(1) * time.Second
}
+ go RunTrashWorker(trashq)
trashq.ReplaceQueue(trashList)
time.Sleep(10 * time.Millisecond) // give a moment to finish processing the list
@@ -282,6 +274,7 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
}
// Done
+ permission_ttl = actual_permission_ttl
trashq.Close()
KeepVM.Quit()
}
commit b01138f2ed3b666fd6075a6d77caddb5d5ca91e1
Merge: fed6d3b 2372145
Author: Radhika Chippada <radhika at curoverse.com>
Date: Mon Mar 23 10:35:34 2015 -0400
Merge branch 'master' of git.curoverse.com:arvados into 3762-delete-list-worker
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list