[ARVADOS] updated: b346f8754f5d4840ad80aa97bd910f3a6844b413

git at public.curoverse.com git at public.curoverse.com
Tue Apr 15 17:20:19 EDT 2014


Summary of changes:
 .../app/controllers/application_controller.rb      |    1 +
 apps/workbench/app/controllers/users_controller.rb |    1 +
 apps/workbench/config/application.default.yml      |    1 +
 .../test/functional/users_controller_test.rb       |   16 +++
 apps/workbench/test/integration/logins_test.rb     |    8 ++
 apps/workbench/test/integration_helper.rb          |   57 +---------
 apps/workbench/test/test_helper.rb                 |   78 +++++++++++++
 services/api/Gemfile                               |    2 +
 services/api/Gemfile.lock                          |  116 ++++++++++----------
 .../api/app/controllers/application_controller.rb  |   10 ++
 .../app/controllers/arvados/v1/jobs_controller.rb  |    2 +-
 .../app/controllers/arvados/v1/users_controller.rb |   13 ++
 services/api/app/mailers/user_notifier.rb          |    8 ++
 .../views/user_notifier/account_is_setup.text.erb  |    6 +
 services/api/config/application.default.yml        |    3 +
 services/api/config/routes.rb                      |    2 +
 services/api/script/setup-new-user.rb              |    9 +-
 .../test/fixtures/api_client_authorizations.yml    |    6 +
 services/api/test/fixtures/jobs.yml                |   13 ++
 .../arvados/v1/job_reuse_controller_test.rb        |   20 +++-
 .../functional/arvados/v1/users_controller_test.rb |   58 ++++++++++
 .../api/test/functional/user_notifier_test.rb      |    2 +-
 services/api/test/unit/user_notifier_test.rb       |   24 ++++
 services/keep/keep.go                              |   41 ++++++--
 services/keep/keep_test.go                         |    7 +
 25 files changed, 379 insertions(+), 125 deletions(-)
 create mode 100644 services/api/app/mailers/user_notifier.rb
 create mode 100644 services/api/app/views/user_notifier/account_is_setup.text.erb
 copy apps/workbench/test/unit/job_task_test.rb => services/api/test/functional/user_notifier_test.rb (61%)
 create mode 100644 services/api/test/unit/user_notifier_test.rb

       via  b346f8754f5d4840ad80aa97bd910f3a6844b413 (commit)
       via  e6815764c940968ed825e8b2e4d984824a68fd23 (commit)
       via  d9f6071c52c33831be88546475480920fe7e1a60 (commit)
       via  c2aeaa729c350a33ec47ae1b012e50a6685ac2d2 (commit)
       via  bb45025e80abc00f8da524d9a78c0bab976d4f3a (commit)
       via  3a76b55556740564b8cfb44c7e430975cca6bf96 (commit)
       via  eeeceb21a479780dfa7d550523ab961f477e39ad (commit)
       via  f9578ab5cacc544dc1a5c3fa0fbea5fd627efdf9 (commit)
       via  9f3211fd8de463cb68febb4e3333721e026605b8 (commit)
       via  c2ccba382871ecad4f9336f250a90a95d8f5b987 (commit)
       via  658465e22d198a453646f0d845a9ccd50fb0a689 (commit)
       via  ee7753f7095f175c095f94794dd3e33f68d3fcc6 (commit)
       via  283dbf37a1b7d32332e295070de48b5e6e459248 (commit)
       via  9c40a72521dbfb4244d48069e4754a342f800492 (commit)
       via  04581ab4e5cea0389be8e641a1123381f0f7217c (commit)
       via  b288ebe08b16204d97c4911106bd5ca57fb2b36b (commit)
      from  ad2984e6eaf029ba7b6e35ae875bb78cfb76dfb6 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit b346f8754f5d4840ad80aa97bd910f3a6844b413
Merge: e681576 c2aeaa7
Author: Tim Pierce <twp at curoverse.com>
Date:   Tue Apr 15 17:20:12 2014 -0400

    Merge branch 'master' into 2449-keep-index-status-handlers


commit e6815764c940968ed825e8b2e4d984824a68fd23
Author: Tim Pierce <twp at curoverse.com>
Date:   Tue Apr 15 17:18:22 2014 -0400

    Add a device_num field to status.json output.
    
    Include the underlying device number (as returned by stat(2)) in
    status.json output, so clients can see if two Keep directories are
    located on the same underlying volume. Refs #2561.

diff --git a/services/keep/keep.go b/services/keep/keep.go
index af8ba2a..91cddcb 100644
--- a/services/keep/keep.go
+++ b/services/keep/keep.go
@@ -226,17 +226,16 @@ func IndexHandler(w http.ResponseWriter, req *http.Request) {
 //     described in a JSON structure.
 //
 //     The data given in a status.json response includes:
-//        time - the time the status was last updated
-//        df   - the output of the most recent `df --block-size=1k`
-//        disk_devices - list of disk device files (i.e. /dev/(s|xv|h)d)
-//        dirs - an object describing Keep volumes, keyed on Keep volume dirs,
-//          each value is an object describing the status of that volume
-//            * status ("full [timestamp]" or "ok [timestamp]")
-//            * last_error
-//            * last_error_time
+//        volumes - a list of Keep volumes currently in use by this server
+//          each volume is an object with the following fields:
+//            * mount_point
+//            * device_num (an integer identifying the underlying filesystem)
+//            * bytes_free
+//            * bytes_used
 //
 type VolumeStatus struct {
 	MountPoint string `json:"mount_point"`
+	DeviceNum  uint64 `json:"device_num"`
 	BytesFree  uint64 `json:"bytes_free"`
 	BytesUsed  uint64 `json:"bytes_used"`
 }
@@ -275,6 +274,14 @@ func GetNodeStatus() *NodeStatus {
 //
 func GetVolumeStatus(volume string) *VolumeStatus {
 	var fs syscall.Statfs_t
+	var devnum uint64
+
+	if fi, err := os.Stat(volume); err == nil {
+		devnum = fi.Sys().(*syscall.Stat_t).Dev
+	} else {
+		log.Printf("GetVolumeStatus: os.Stat: %s\n", err)
+		return nil
+	}
 
 	err := syscall.Statfs(volume, &fs)
 	if err != nil {
@@ -286,7 +293,7 @@ func GetVolumeStatus(volume string) *VolumeStatus {
 	// uses fs.Blocks - fs.Bfree.
 	free := fs.Bavail * uint64(fs.Bsize)
 	used := (fs.Blocks - fs.Bfree) * uint64(fs.Bsize)
-	return &VolumeStatus{volume, free, used}
+	return &VolumeStatus{volume, devnum, free, used}
 }
 
 // IndexLocators
diff --git a/services/keep/keep_test.go b/services/keep/keep_test.go
index 2d1916e..97fa1c7 100644
--- a/services/keep/keep_test.go
+++ b/services/keep/keep_test.go
@@ -340,6 +340,9 @@ func TestNodeStatus(t *testing.T) {
 		if mtp != vol {
 			t.Errorf("GetNodeStatus mount_point %s != KeepVolume %s", mtp, vol)
 		}
+		if volinfo.DeviceNum == 0 {
+			t.Errorf("uninitialized device_num in %v", volinfo)
+		}
 		if volinfo.BytesFree == 0 {
 			t.Errorf("uninitialized bytes_free in %v", volinfo)
 		}

commit d9f6071c52c33831be88546475480920fe7e1a60
Author: Tim Pierce <twp at curoverse.com>
Date:   Tue Apr 15 16:46:57 2014 -0400

    Added IsValidLocator to filter /index output.
    
    IndexHandler should skip metadata files and anything else that isn't a
    valid locator, so call IsValidLocator before including the file in the
    index output. Refs #2561.

diff --git a/services/keep/keep.go b/services/keep/keep.go
index 7323c89..af8ba2a 100644
--- a/services/keep/keep.go
+++ b/services/keep/keep.go
@@ -15,6 +15,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"syscall"
@@ -325,6 +326,12 @@ func IndexLocators(prefix string) string {
 					!strings.HasPrefix(prefix, locator) {
 					return filepath.SkipDir
 				}
+				// Skip any file that is not apparently a locator, e.g. .meta files
+				if is_valid, err := IsValidLocator(locator); err != nil {
+					return err
+				} else if !is_valid {
+					return nil
+				}
 				// Print filenames beginning with prefix
 				if !info.IsDir() && strings.HasPrefix(locator, prefix) {
 					output = output + fmt.Sprintf(
@@ -544,3 +551,12 @@ func ReadAtMost(r io.Reader, maxbytes int) ([]byte, error) {
 	}
 	return buf, err
 }
+
+// IsValidLocator
+//     Return true if the specified string is a valid Keep locator.
+//     When Keep is extended to support hash types other than MD5,
+//     this should be updated to cover those as well.
+//
+func IsValidLocator(loc string) (bool, error) {
+	return regexp.MatchString(`^[0-9a-f]{32}$`, loc)
+}
diff --git a/services/keep/keep_test.go b/services/keep/keep_test.go
index 05eb410..2d1916e 100644
--- a/services/keep/keep_test.go
+++ b/services/keep/keep_test.go
@@ -296,10 +296,14 @@ func TestIndex(t *testing.T) {
 	defer teardown()
 
 	// Set up Keep volumes and populate them.
+	// Include multiple blocks on different volumes, and
+	// some metadata files.
 	KeepVolumes = setup(t, 2)
 	store(t, KeepVolumes[0], TEST_HASH, TEST_BLOCK)
 	store(t, KeepVolumes[1], TEST_HASH_2, TEST_BLOCK_2)
 	store(t, KeepVolumes[0], TEST_HASH_3, TEST_BLOCK_3)
+	store(t, KeepVolumes[0], TEST_HASH+".meta", []byte("metadata"))
+	store(t, KeepVolumes[1], TEST_HASH_2+".meta", []byte("metadata"))
 
 	index := IndexLocators("")
 	expected := `^` + TEST_HASH + `\+\d+ \d+\n` +

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list