[ARVADOS] updated: 8678b8542a1e73c9a7861a9b8ba7488a1a32c1c4
git at public.curoverse.com
git at public.curoverse.com
Wed Sep 9 15:12:12 EDT 2015
Summary of changes:
sdk/python/tests/run_test_server.py | 8 ++--
services/datamanager/datamanager.go | 33 ++++-----------
services/datamanager/datamanager_test.go | 51 ++++++++--------------
services/datamanager/keep/keep.go | 72 ++++++--------------------------
4 files changed, 40 insertions(+), 124 deletions(-)
via 8678b8542a1e73c9a7861a9b8ba7488a1a32c1c4 (commit)
via 20b7c375715bed99ccfe2dd12ed816b545e94324 (commit)
via 6f0362fb0f425236859deaa55902d3082dad32c0 (commit)
via 46f82c4bfcece82b430455fbefaa2d46f6abf647 (commit)
via b4866c16547c6c3196d5d8327686986e8997c187 (commit)
from 902c3317440f81077ea561b621c8d46dcb689013 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8678b8542a1e73c9a7861a9b8ba7488a1a32c1c4
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 9 15:09:46 2015 -0400
6260: Simplify verifyBlocks logic.
diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 18b302a..c76d481 100644
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -207,31 +207,24 @@ func getBlockIndexes(t *testing.T) [][]string {
func verifyBlocks(t *testing.T, notExpected []string, expected []string) {
blocks := getBlockIndexes(t)
+
for _, block := range notExpected {
- for i := 0; i < len(blocks); i++ {
- exists := valueInArray(block, blocks[i])
- if exists {
- t.Fatalf("Found unexpected block in index %s", block)
+ for _, idx := range blocks {
+ if valueInArray(block, idx) {
+ t.Fatalf("Found unexpected block %s", block)
}
}
}
- // var blockExists [][]string
- blockExists := make(map[string][]string)
for _, block := range expected {
- var blockArray []string
- for i := 0; i < len(blocks); i++ {
- exists := valueInArray(block, blocks[i])
- if exists {
- blockArray = append(blockArray, block)
+ nFound := 0
+ for _, idx := range blocks {
+ if valueInArray(block, idx) {
+ nFound++
}
}
- blockExists[block] = blockArray
- }
-
- for _, block := range expected {
- if blockExists[block] == nil || len(blockExists[block]) < 2 {
- t.Fatalf("Expected to find two replicas for block %s; found %d", block, len(blockExists[block]))
+ if nFound < 2 {
+ t.Fatalf("Found %d replicas of block %s, expected >= 2", nFound, block)
}
}
}
commit 20b7c375715bed99ccfe2dd12ed816b545e94324
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 9 15:09:15 2015 -0400
6260: Remove test for impossible condition.
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index ce316f1..1a75bb1 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -94,11 +94,6 @@ func GetKeepServersAndSummarize(params GetKeepServersParams) (results ReadServer
}
func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
- if ¶ms.Client == nil {
- log.Fatalf("params.Client passed to GetKeepServers() should " +
- "contain a valid ArvadosClient, but instead it is nil.")
- }
-
sdkParams := arvadosclient.Dict{
"filters": [][]string{[]string{"service_type", "=", "disk"}},
}
commit 6f0362fb0f425236859deaa55902d3082dad32c0
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 9 15:09:05 2015 -0400
6260: Fix up error messages.
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 3fb135c..4e3b4e9 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -44,7 +44,7 @@ func main() {
if minutesBetweenRuns == 0 {
err := singlerun(makeArvadosClient())
if err != nil {
- log.Fatalf("Got an error: %v", err)
+ log.Fatalf("singlerun: %v", err)
}
} else {
waitTime := time.Minute * time.Duration(minutesBetweenRuns)
@@ -52,7 +52,7 @@ func main() {
log.Println("Beginning Run")
err := singlerun(makeArvadosClient())
if err != nil {
- log.Printf("Got an error: %v", err)
+ log.Printf("singlerun: %v", err)
}
log.Printf("Sleeping for %d minutes", minutesBetweenRuns)
time.Sleep(waitTime)
@@ -70,12 +70,10 @@ func makeArvadosClient() arvadosclient.ArvadosClient {
func singlerun(arv arvadosclient.ArvadosClient) error {
var err error
- if is_admin, err := util.UserIsAdmin(arv); err != nil {
- log.Printf("Error querying current arvados user %s", err.Error())
- return err
- } else if !is_admin {
- log.Printf("Current user is not an admin. Datamanager can only be run by admins.")
- return errors.New("Current user is not an admin. Datamanager can only be run by admins.")
+ if isAdmin, err := util.UserIsAdmin(arv); err != nil {
+ return errors.New("Error verifying admin token: " + err.Error())
+ } else if !isAdmin {
+ return errors.New("Current user is not an admin. Datamanager requires a privileged token.")
}
var arvLogger *logger.Logger
commit 46f82c4bfcece82b430455fbefaa2d46f6abf647
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 9 15:08:31 2015 -0400
6260: Remove extra hyphen from command line args.
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 4421262..3fa1ccc 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -330,14 +330,14 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
if not blob_signing_key:
blob_signing_key = 'zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc'
with open(os.path.join(TEST_TMPDIR, "keep.blob_signing_key"), "w") as f:
- keep_args['--blob-signing-key-file'] = f.name
+ keep_args['-blob-signing-key-file'] = f.name
f.write(blob_signing_key)
if enforce_permissions:
- keep_args['--enforce-permissions'] = 'true'
+ keep_args['-enforce-permissions'] = 'true'
with open(os.path.join(TEST_TMPDIR, "keep.data-manager-token-file"), "w") as f:
- keep_args['--data-manager-token-file'] = f.name
+ keep_args['-data-manager-token-file'] = f.name
f.write(os.environ['ARVADOS_API_TOKEN'])
- keep_args['--never-delete'] = 'false'
+ keep_args['-never-delete'] = 'false'
api = arvados.api(
version='v1',
commit b4866c16547c6c3196d5d8327686986e8997c187
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 9 15:08:09 2015 -0400
6260: Just use the token loaded by the SDK (ARVADOS_API_TOKEN),
instead of requiring a token separately on the command line.
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index 9cfd2ee..3fb135c 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -68,8 +68,6 @@ func makeArvadosClient() arvadosclient.ArvadosClient {
return arv
}
-var dataManagerToken string
-
func singlerun(arv arvadosclient.ArvadosClient) error {
var err error
if is_admin, err := util.UserIsAdmin(arv); err != nil {
@@ -93,21 +91,6 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
arvLogger.AddWriteHook(loggerutil.LogMemoryAlloc)
}
- // Verify that datamanager token belongs to an admin user
- if dataManagerToken == "" {
- dataManagerToken = keep.GetDataManagerToken(arvLogger)
- }
- origArvToken := arv.ApiToken
- arv.ApiToken = dataManagerToken
- if is_admin, err := util.UserIsAdmin(arv); err != nil {
- log.Printf("Error querying arvados user for data manager token %s", err.Error())
- return err
- } else if !is_admin {
- log.Printf("Datamanager token does not belong to an admin user.")
- return errors.New("Datamanager token does not belong to an admin user.")
- }
- arv.ApiToken = origArvToken
-
var (
dataFetcher summary.DataFetcher
readCollections collection.ReadCollections
@@ -178,7 +161,7 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
if trashErr != nil {
return err
} else {
- keep.SendTrashLists(dataManagerToken, kc, trashLists)
+ keep.SendTrashLists(kc, trashLists)
}
return nil
diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 2ab1a48..18b302a 100644
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -6,7 +6,6 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
- "git.curoverse.com/arvados.git/services/datamanager/keep"
"io/ioutil"
"net/http"
"os"
@@ -17,7 +16,10 @@ import (
"time"
)
-const ACTIVE_USER_TOKEN = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+const (
+ ActiveUserToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+ AdminToken = "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h"
+)
var arv arvadosclient.ArvadosClient
var keepClient *keepclient.KeepClient
@@ -172,7 +174,7 @@ func getBlockIndexesForServer(t *testing.T, i int) []string {
path := keepServers[i] + "/index"
client := http.Client{}
req, err := http.NewRequest("GET", path, nil)
- req.Header.Add("Authorization", "OAuth2 "+keep.GetDataManagerToken(nil))
+ req.Header.Add("Authorization", "OAuth2 " + AdminToken)
req.Header.Add("Content-Type", "application/octet-stream")
resp, err := client.Do(req)
defer resp.Body.Close()
@@ -301,7 +303,7 @@ func backdateBlocks(t *testing.T, oldUnusedBlockLocators []string) {
func getStatus(t *testing.T, path string) interface{} {
client := http.Client{}
req, err := http.NewRequest("GET", path, nil)
- req.Header.Add("Authorization", "OAuth2 "+keep.GetDataManagerToken(nil))
+ req.Header.Add("Authorization", "OAuth2 " + AdminToken)
req.Header.Add("Content-Type", "application/octet-stream")
resp, err := client.Do(req)
defer resp.Body.Close()
@@ -531,22 +533,10 @@ func TestRunDatamanagerAsNonAdminUser(t *testing.T) {
defer TearDownDataManagerTest(t)
SetupDataManagerTest(t)
- arv.ApiToken = ACTIVE_USER_TOKEN
+ arv.ApiToken = ActiveUserToken
err := singlerun(arv)
if err == nil {
t.Fatalf("Expected error during singlerun as non-admin user")
}
}
-
-func TestRunDatamanagerWithNonAdminDataManagerToken(t *testing.T) {
- defer TearDownDataManagerTest(t)
- SetupDataManagerTest(t)
-
- dataManagerToken = ACTIVE_USER_TOKEN
-
- err := singlerun(arv)
- if err == nil {
- t.Fatalf("Expected error during singlerun with non-admin user token as datamanager token")
- }
-}
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 0e3cc1d..ce316f1 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -6,7 +6,6 @@ import (
"bufio"
"encoding/json"
"errors"
- "flag"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/blockdigest"
@@ -19,7 +18,6 @@ import (
"net/http"
"strconv"
"strings"
- "sync"
"time"
)
@@ -71,21 +69,6 @@ type KeepServiceList struct {
KeepServers []ServerAddress `json:"items"`
}
-var (
- // Don't access the token directly, use getDataManagerToken() to
- // make sure it's been read.
- dataManagerToken string
- dataManagerTokenFile string
- dataManagerTokenFileReadOnce sync.Once
-)
-
-func init() {
- flag.StringVar(&dataManagerTokenFile,
- "data-manager-token-file",
- "",
- "File with the API token we should use to contact keep servers.")
-}
-
// TODO(misha): Change this to include the UUID as well.
func (s ServerAddress) String() string {
return s.URL()
@@ -99,28 +82,6 @@ func (s ServerAddress) URL() string {
}
}
-func GetDataManagerToken(arvLogger *logger.Logger) string {
- readDataManagerToken := func() {
- if dataManagerTokenFile == "" {
- flag.Usage()
- loggerutil.FatalWithMessage(arvLogger,
- "Data Manager Token needed, but data manager token file not specified.")
- } else {
- rawRead, err := ioutil.ReadFile(dataManagerTokenFile)
- if err != nil {
- loggerutil.FatalWithMessage(arvLogger,
- fmt.Sprintf("Unexpected error reading token file %s: %v",
- dataManagerTokenFile,
- err))
- }
- dataManagerToken = strings.TrimSpace(string(rawRead))
- }
- }
-
- dataManagerTokenFileReadOnce.Do(readDataManagerToken)
- return dataManagerToken
-}
-
func GetKeepServersAndSummarize(params GetKeepServersParams) (results ReadServers) {
results = GetKeepServers(params)
log.Printf("Returned %d keep disks", len(results.ServerToContents))
@@ -177,9 +138,6 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
log.Printf("Got Server Addresses: %v", results)
- // This is safe for concurrent use
- client := http.Client{}
-
// Send off all the index requests concurrently
responseChan := make(chan ServerResponse)
for _, keepServer := range sdkResponse.KeepServers {
@@ -192,7 +150,7 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
go func(keepServer ServerAddress) {
responseChan <- GetServerContents(params.Logger,
keepServer,
- client)
+ params.Client)
}(keepServer)
}
@@ -220,12 +178,12 @@ func GetKeepServers(params GetKeepServersParams) (results ReadServers) {
func GetServerContents(arvLogger *logger.Logger,
keepServer ServerAddress,
- client http.Client) (response ServerResponse) {
+ arv arvadosclient.ArvadosClient) (response ServerResponse) {
- GetServerStatus(arvLogger, keepServer, client)
+ GetServerStatus(arvLogger, keepServer, arv)
- req := CreateIndexRequest(arvLogger, keepServer)
- resp, err := client.Do(req)
+ req := CreateIndexRequest(arvLogger, keepServer, arv)
+ resp, err := arv.Client.Do(req)
if err != nil {
loggerutil.FatalWithMessage(arvLogger,
fmt.Sprintf("Error fetching %s: %v. Response was %+v",
@@ -239,7 +197,7 @@ func GetServerContents(arvLogger *logger.Logger,
func GetServerStatus(arvLogger *logger.Logger,
keepServer ServerAddress,
- client http.Client) {
+ arv arvadosclient.ArvadosClient) {
url := fmt.Sprintf("http://%s:%d/status.json",
keepServer.Host,
keepServer.Port)
@@ -257,7 +215,7 @@ func GetServerStatus(arvLogger *logger.Logger,
})
}
- resp, err := client.Get(url)
+ resp, err := arv.Client.Get(url)
if err != nil {
loggerutil.FatalWithMessage(arvLogger,
fmt.Sprintf("Error getting keep status from %s: %v", url, err))
@@ -289,7 +247,8 @@ func GetServerStatus(arvLogger *logger.Logger,
}
func CreateIndexRequest(arvLogger *logger.Logger,
- keepServer ServerAddress) (req *http.Request) {
+ keepServer ServerAddress,
+ arv arvadosclient.ArvadosClient) (req *http.Request) {
url := fmt.Sprintf("http://%s:%d/index", keepServer.Host, keepServer.Port)
log.Println("About to fetch keep server contents from " + url)
@@ -308,8 +267,7 @@ func CreateIndexRequest(arvLogger *logger.Logger,
fmt.Sprintf("Error building http request for %s: %v", url, err))
}
- req.Header.Add("Authorization",
- fmt.Sprintf("OAuth2 %s", GetDataManagerToken(arvLogger)))
+ req.Header.Add("Authorization", "OAuth2 " + arv.ApiToken)
return
}
@@ -462,7 +420,7 @@ type TrashRequest struct {
type TrashList []TrashRequest
-func SendTrashLists(dataManagerToken string, kc *keepclient.KeepClient, spl map[string]TrashList) (errs []error) {
+func SendTrashLists(kc *keepclient.KeepClient, spl map[string]TrashList) (errs []error) {
count := 0
barrier := make(chan error)
@@ -487,8 +445,7 @@ func SendTrashLists(dataManagerToken string, kc *keepclient.KeepClient, spl map[
return
}
- // Add api token header
- req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", dataManagerToken))
+ req.Header.Add("Authorization", "OAuth2 " + kc.Arvados.ApiToken)
// Make the request
var resp *http.Response
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list