[ARVADOS] updated: 5251261a5fbf8ff746f8a2ff2226d8af6ff6c12d
git at public.curoverse.com
git at public.curoverse.com
Tue Dec 1 10:42:31 EST 2015
Summary of changes:
services/datamanager/datamanager.go | 11 ++--
services/datamanager/keep/keep.go | 100 +++++++++++++++---------------
services/datamanager/summary/pull_list.go | 46 +++++++-------
3 files changed, 81 insertions(+), 76 deletions(-)
via 5251261a5fbf8ff746f8a2ff2226d8af6ff6c12d (commit)
from d005c38bea17107e7be3c5338ad31ba54085df61 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5251261a5fbf8ff746f8a2ff2226d8af6ff6c12d
Author: radhika <radhika at curoverse.com>
Date: Tue Dec 1 10:41:38 2015 -0500
7748: when dryRun continue and leave the regular flow untouched.
diff --git a/services/datamanager/datamanager.go b/services/datamanager/datamanager.go
index ba7446e..c1a5cbe 100644
--- a/services/datamanager/datamanager.go
+++ b/services/datamanager/datamanager.go
@@ -138,6 +138,11 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
rlbss.Count)
}
+ kc, err := keepclient.MakeKeepClient(&arv)
+ if err != nil {
+ return fmt.Errorf("Error setting up keep client %v", err.Error())
+ }
+
// Log that we're finished. We force the recording, since go will
// not wait for the write timer before exiting.
if arvLogger != nil {
@@ -151,12 +156,6 @@ func singlerun(arv arvadosclient.ArvadosClient) error {
})
}
- // Not dry-run; issue changes to keepstore
- kc, err := keepclient.MakeKeepClient(&arv)
- if err != nil {
- return fmt.Errorf("Error setting up keep client %v", err.Error())
- }
-
pullServers := summary.ComputePullServers(kc,
&keepServerInfo,
readCollections.BlockToDesiredReplication,
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index c49a021..cbc4047 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -456,57 +456,59 @@ func SendTrashLists(arvLogger *logger.Logger, kc *keepclient.KeepClient, spl map
client := kc.Client
for url, v := range spl {
+ // We need a local variable because Update doesn't call our mutator func until later,
+ // when our list variable might have been reused by the next loop iteration.
+ if arvLogger != nil {
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ trashListInfo := logger.GetOrCreateMap(p, "trash_list_len")
+ trashListInfo[url] = len(v)
+ })
+ }
+
if dryRun {
- if arvLogger != nil {
- for url, v := range spl {
- arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
- trashListInfo := logger.GetOrCreateMap(p, "trash_list")
- trashListInfo["started_at"] = time.Now()
- trashListInfo[url] = v
- })
- }
- }
- } else {
- count++
- log.Printf("Sending trash list to %v", url)
-
- go (func(url string, v TrashList) {
- pipeReader, pipeWriter := io.Pipe()
- go (func() {
- enc := json.NewEncoder(pipeWriter)
- enc.Encode(v)
- pipeWriter.Close()
- })()
-
- req, err := http.NewRequest("PUT", fmt.Sprintf("%s/trash", url), pipeReader)
- if err != nil {
- log.Printf("Error creating trash list request for %v error: %v", url, err.Error())
- barrier <- err
- return
- }
-
- req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
-
- // Make the request
- var resp *http.Response
- if resp, err = client.Do(req); err != nil {
- log.Printf("Error sending trash list to %v error: %v", url, err.Error())
- barrier <- err
- return
- }
-
- log.Printf("Sent trash list to %v: response was HTTP %v", url, resp.Status)
-
- io.Copy(ioutil.Discard, resp.Body)
- resp.Body.Close()
-
- if resp.StatusCode != 200 {
- barrier <- errors.New(fmt.Sprintf("Got HTTP code %v", resp.StatusCode))
- } else {
- barrier <- nil
- }
- })(url, v)
+ log.Printf("dry run, not sending trash list to service %s with %d blocks", url, len(v))
+ continue
}
+
+ count++
+ log.Printf("Sending trash list to %v", url)
+
+ go (func(url string, v TrashList) {
+ pipeReader, pipeWriter := io.Pipe()
+ go (func() {
+ enc := json.NewEncoder(pipeWriter)
+ enc.Encode(v)
+ pipeWriter.Close()
+ })()
+
+ req, err := http.NewRequest("PUT", fmt.Sprintf("%s/trash", url), pipeReader)
+ if err != nil {
+ log.Printf("Error creating trash list request for %v error: %v", url, err.Error())
+ barrier <- err
+ return
+ }
+
+ req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
+
+ // Make the request
+ var resp *http.Response
+ if resp, err = client.Do(req); err != nil {
+ log.Printf("Error sending trash list to %v error: %v", url, err.Error())
+ barrier <- err
+ return
+ }
+
+ log.Printf("Sent trash list to %v: response was HTTP %v", url, resp.Status)
+
+ io.Copy(ioutil.Discard, resp.Body)
+ resp.Body.Close()
+
+ if resp.StatusCode != 200 {
+ barrier <- errors.New(fmt.Sprintf("Got HTTP code %v", resp.StatusCode))
+ } else {
+ barrier <- nil
+ }
+ })(url, v)
}
for i := 0; i < count; i++ {
diff --git a/services/datamanager/summary/pull_list.go b/services/datamanager/summary/pull_list.go
index 6058616..34149e8 100644
--- a/services/datamanager/summary/pull_list.go
+++ b/services/datamanager/summary/pull_list.go
@@ -12,7 +12,6 @@ import (
"log"
"os"
"strings"
- "time"
)
// Locator is a block digest
@@ -181,29 +180,34 @@ func WritePullLists(arvLogger *logger.Logger,
r := strings.NewReplacer(":", ".")
for host, list := range pullLists {
+ if arvLogger != nil {
+ // We need a local variable because Update doesn't call our mutator func until later,
+ // when our list variable might have been reused by the next loop iteration.
+ arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
+ pullListInfo := logger.GetOrCreateMap(p, "pull_list_len")
+ pullListInfo[host] = len(list)
+ })
+ }
+
if dryRun {
- if arvLogger != nil {
- arvLogger.Update(func(p map[string]interface{}, e map[string]interface{}) {
- pullListInfo := logger.GetOrCreateMap(p, "pull_list")
- pullListInfo["started_at"] = time.Now()
- pullListInfo[host] = list
- })
- }
- } else {
- filename := fmt.Sprintf("pull_list.%s", r.Replace(RemoveProtocolPrefix(host)))
- pullListFile, err := os.Create(filename)
- if err != nil {
- return err
- }
- defer pullListFile.Close()
+ log.Print("dry run, not sending pull list to service %s with %d blocks", host, len(list))
+ continue
+ }
- enc := json.NewEncoder(pullListFile)
- err = enc.Encode(list)
- if err != nil {
- return err
- }
- log.Printf("Wrote pull list to %s.", filename)
+ filename := fmt.Sprintf("pull_list.%s", r.Replace(RemoveProtocolPrefix(host)))
+ pullListFile, err := os.Create(filename)
+ if err != nil {
+ return err
+ }
+ defer pullListFile.Close()
+
+ enc := json.NewEncoder(pullListFile)
+ err = enc.Encode(list)
+ if err != nil {
+ return err
}
+ log.Printf("Wrote pull list to %s.", filename)
}
+
return nil
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list