[ARVADOS] updated: 3c554a995e2b4f716106876ffbd311e87cce4091

git at public.curoverse.com git at public.curoverse.com
Thu Apr 10 18:45:33 EDT 2014


Summary of changes:
 sdk/cli/bin/crunch-job                  |    1 +
 services/api/app/models/user.rb         |   14 ++--
 services/api/config/database.yml.sample |    4 +-
 services/api/test/fixtures/jobs.yml     |    2 +
 services/api/test/unit/user_test.rb     |   10 ++-
 services/keep/keep.go                   |  146 ++++++++++++++++++++++---------
 services/keep/keep_test.go              |   48 +++++++---
 7 files changed, 159 insertions(+), 66 deletions(-)

       via  3c554a995e2b4f716106876ffbd311e87cce4091 (commit)
       via  0a09da628d4d7f1ced2daaa55603bea940b9211c (commit)
       via  b452636351b19ed5641a20eabc68881bd8905ca4 (commit)
       via  bb3a7e08a90ea7826c0b0e1289c64b395b472151 (commit)
       via  e35ed29187d83ebd4cbc493b9251119013825ac1 (commit)
       via  71b0d0fb51e3c54a7959f51fd4dbf523fbaf57db (commit)
       via  6ea2e62b70a015226c4f3361ab3591509100a820 (commit)
       via  e4b0ff638bb41ce55ab3770c4f2b7f744d653aac (commit)
       via  50ee9817061629f9ffe2568f937149f6e877df04 (commit)
      from  c7b155c06452315396ec98f42cf5b502be38b221 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 3c554a995e2b4f716106876ffbd311e87cce4091
Merge: 0a09da6 b452636
Author: Tim Pierce <twp at curoverse.com>
Date:   Thu Apr 10 18:45:00 2014 -0400

    Merge branch '2449-keep-write-blocks' into 2449-keep-index-status-handlers
    
    Conflicts:
    	services/keep/keep.go

diff --cc services/keep/keep_test.go
index 2aa8aff,3f09e90..250c556
--- a/services/keep/keep_test.go
+++ b/services/keep/keep_test.go
@@@ -11,17 -11,19 +12,28 @@@ import 
  
  var TEST_BLOCK = []byte("The quick brown fox jumps over the lazy dog.")
  var TEST_HASH = "e4d909c290d0fb1ca068ffaddf22cbd0"
 +
 +var TEST_BLOCK_2 = []byte("Pack my box with five dozen liquor jugs.")
 +var TEST_HASH_2 = "f15ac516f788aec4f30932ffb6395c39"
 +
 +var TEST_BLOCK_3 = []byte("Now is the time for all good men to come to the aid of their country.")
 +var TEST_HASH_3 = "eed29bbffbc2dbe5e5ee0bb71888e61f"
 +
 +// BAD_BLOCK is used to test collisions and corruption.
 +// It must not match any test hashes.
  var BAD_BLOCK = []byte("The magic words are squeamish ossifrage.")
  
+ // TODO(twp): Tests still to be written
+ //
+ //   * PutBlockCollision
+ //       - test that PutBlock(BLOCK, HASH) reports a collision. HASH must
+ //         be present in Keep and identify a block which sums to HASH but
+ //         which does not match BLOCK. (Requires an interface to mock MD5.)
+ //
+ //   * PutBlockFull
+ //       - test that PutBlock returns 503 Full if the filesystem is full.
+ //         (must mock FreeDiskSpace or Statfs? use a tmpfs?)
+ 
  // ========================================
  // GetBlock tests.
  // ========================================

commit 0a09da628d4d7f1ced2daaa55603bea940b9211c
Author: Tim Pierce <twp at curoverse.com>
Date:   Thu Apr 10 18:43:01 2014 -0400

    Added status.json handler. (refs #2561)

diff --git a/services/keep/keep.go b/services/keep/keep.go
index adffaa4..b96afff 100644
--- a/services/keep/keep.go
+++ b/services/keep/keep.go
@@ -10,10 +10,10 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 )
 
@@ -69,6 +69,7 @@ func main() {
 	rest.HandleFunc(`/{hash:[0-9a-f]{32}}`, PutBlockHandler).Methods("PUT")
 	rest.HandleFunc(`/index`, IndexHandler).Methods("GET", "HEAD")
 	rest.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, IndexHandler).Methods("GET", "HEAD")
+	rest.HandleFunc(`/status\.json`, StatusHandler).Methods("GET", "HEAD")
 
 	// Tell the built-in HTTP server to direct all requests to the REST
 	// router.
@@ -154,6 +155,80 @@ func IndexHandler(w http.ResponseWriter, req *http.Request) {
 	w.Write([]byte(index))
 }
 
+// StatusHandler
+//     Responds to /status.json requests with the current node status,
+//     described in a JSON structure.
+//
+//     The data given in a status.json response includes:
+//        time - the time the status was last updated
+//        df   - the output of the most recent `df --block-size=1k`
+//        disk_devices - list of disk device files (i.e. /dev/(s|xv|h)d)
+//        dirs - an object describing Keep volumes, keyed on Keep volume dirs,
+//          each value is an object describing the status of that volume
+//            * status ("full [timestamp]" or "ok [timestamp]")
+//            * last_error
+//            * last_error_time
+//
+type VolumeStatus struct {
+	Space       string
+	LastErr     string
+	LastErrTime time.Time
+}
+
+type NodeStatus struct {
+	LastUpdate time.Time
+	DfOutput   string
+	DiskDev    []string
+	Volumes    map[string]VolumeStatus
+}
+
+func StatusHandler(w http.ResponseWriter, req *http.Request) {
+	st := new(NodeStatus)
+	st.LastUpdate = time.Now()
+
+	// Get a list of disk devices on this system.
+	st.DiskDev = make([]string, 1)
+	if devdir, err := os.Open("/dev"); err != nil {
+		log.Printf("StatusHandler: opening /dev: %s\n", err)
+	} else {
+		devs, err := devdir.Readdirnames(0)
+		if err == nil {
+			for _, d := range devs {
+				if strings.HasPrefix(d, "sd") ||
+					strings.HasPrefix(d, "hd") ||
+					strings.HasPrefix(d, "xvd") {
+					st.DiskDev = append(st.DiskDev, d)
+				}
+			}
+		} else {
+			log.Printf("Readdirnames: %s", err)
+		}
+	}
+
+	for _, vol := range KeepVolumes {
+		st.Volumes[vol] = GetVolumeStatus(vol)
+	}
+}
+
+// GetVolumeStatus
+//     Returns a VolumeStatus describing the requested volume.
+func GetVolumeStatus(volume string) VolumeStatus {
+	var isfull, lasterr string
+	var lasterrtime time.Time
+
+	if IsFull(volume) {
+		isfull = fmt.Sprintf("full %d", time.Now().Unix())
+	} else {
+		isfull = fmt.Sprintf("ok %d", time.Now().Unix())
+	}
+
+	// Not implemented yet
+	lasterr = ""
+	lasterrtime = time.Unix(0, 0)
+
+	return VolumeStatus{isfull, lasterr, lasterrtime}
+}
+
 // IndexLocators
 //     Returns a string containing a list of locator ids found on this
 //     Keep server.  If {prefix} is given, return only those locator
@@ -373,32 +448,13 @@ func IsFull(volume string) (isFull bool) {
 //     Returns the amount of available disk space on VOLUME,
 //     as a number of 1k blocks.
 //
-func FreeDiskSpace(volume string) (free int, err error) {
-	// Run df to find out how much disk space is left.
-	cmd := exec.Command("df", "--block-size=1k", volume)
-	stdout, perr := cmd.StdoutPipe()
-	if perr != nil {
-		return 0, perr
+func FreeDiskSpace(volume string) (free uint64, err error) {
+	var fs syscall.Statfs_t
+	err = syscall.Statfs(volume, &fs)
+	if err == nil {
+		// Statfs output is not guaranteed to measure free
+		// space in terms of 1K blocks.
+		free = fs.Bavail * uint64(fs.Bsize) / 1024
 	}
-	scanner := bufio.NewScanner(stdout)
-	if perr := cmd.Start(); err != nil {
-		return 0, perr
-	}
-
-	scanner.Scan() // skip header line of df output
-	scanner.Scan()
-
-	f := strings.Fields(scanner.Text())
-	if avail, err := strconv.Atoi(f[3]); err == nil {
-		free = avail
-	} else {
-		err = errors.New("bad df format: " + scanner.Text())
-	}
-
-	// Flush the df output and shut it down cleanly.
-	for scanner.Scan() {
-	}
-	cmd.Wait()
-
 	return
 }

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list