[ARVADOS] updated: 5c6c6946b034b821446a3657ca065b5646e0f104
git at public.curoverse.com
git at public.curoverse.com
Tue Sep 1 17:39:32 EDT 2015
Summary of changes:
.gitignore | 3 +
.../install-arv-git-httpd.html.textile.liquid | 6 +-
doc/install/install-keepproxy.html.textile.liquid | 10 +-
.../install-shell-server.html.textile.liquid | 95 ++++++++++++
.../install-workbench-app.html.textile.liquid | 46 +++---
doc/user/topics/arv-docker.html.textile.liquid | 2 +-
.../arvados/v1/virtual_machines_controller.rb | 18 ++-
services/api/lib/salvage_collection.rb | 94 ++++++++++++
services/api/script/salvage_collection.rb | 26 ++++
.../arvados/v1/virtual_machines_controller_test.rb | 7 +
services/api/test/unit/salvage_collection_test.rb | 165 +++++++++++++++++++++
services/arv-git-httpd/auth_handler.go | 7 +-
services/arv-git-httpd/doc.go | 26 ++--
services/arv-git-httpd/git_handler.go | 59 ++++++++
services/arv-git-httpd/git_handler_test.go | 56 +++++++
services/arv-git-httpd/main.go | 2 +-
services/arv-git-httpd/server.go | 13 +-
services/arv-git-httpd/server_test.go | 2 +-
services/datamanager/datamanager_test.go | 102 +++++++++----
services/keepstore/volume_unix.go | 72 ++++++---
services/keepstore/volume_unix_test.go | 41 ++++-
{sdk/ruby => services/login-sync}/.gitignore | 2 +-
services/login-sync/Gemfile | 7 +
{sdk/ruby => services/login-sync}/Rakefile | 0
.../login-sync/arvados-login-sync.gemspec | 18 +--
services/login-sync/bin/arvados-login-sync | 111 ++++++++++++++
services/login-sync/test/binstub_new_user/useradd | 9 ++
services/login-sync/test/stubs.rb | 52 +++++++
services/login-sync/test/test_add_user.rb | 37 +++++
.../arvnodeman/computenode/driver/__init__.py | 75 ++++++++--
.../arvnodeman/computenode/driver/azure.py | 71 +++++++++
.../arvnodeman/computenode/driver/ec2.py | 5 -
.../doc/{ec2.example.cfg => azure.example.cfg} | 102 +++++++------
services/nodemanager/setup.py | 5 +-
.../tests/test_computenode_driver_azure.py | 89 +++++++++++
35 files changed, 1244 insertions(+), 191 deletions(-)
create mode 100755 services/api/lib/salvage_collection.rb
create mode 100755 services/api/script/salvage_collection.rb
create mode 100644 services/api/test/unit/salvage_collection_test.rb
create mode 100644 services/arv-git-httpd/git_handler.go
create mode 100644 services/arv-git-httpd/git_handler_test.go
copy {sdk/ruby => services/login-sync}/.gitignore (52%)
create mode 100644 services/login-sync/Gemfile
copy {sdk/ruby => services/login-sync}/Rakefile (100%)
copy sdk/ruby/arvados.gemspec => services/login-sync/arvados-login-sync.gemspec (51%)
create mode 100755 services/login-sync/bin/arvados-login-sync
create mode 100755 services/login-sync/test/binstub_new_user/useradd
create mode 100644 services/login-sync/test/stubs.rb
create mode 100644 services/login-sync/test/test_add_user.rb
create mode 100644 services/nodemanager/arvnodeman/computenode/driver/azure.py
copy services/nodemanager/doc/{ec2.example.cfg => azure.example.cfg} (55%)
create mode 100644 services/nodemanager/tests/test_computenode_driver_azure.py
via 5c6c6946b034b821446a3657ca065b5646e0f104 (commit)
via 505be2eb9ac1f36a5e31b374451193bc1ab7e90e (commit)
via bd4c54298a89dd24b6a457bf4ffd869f5af37c52 (commit)
via 1352b00b037db2ce5de291127b4f1e845bc6b73b (commit)
via 29c5c69ef3c9ceadbea3085f2268bb5ed8496c04 (commit)
via 59b414ff7a625ad3d6e92659b20bdabc6d89e7d4 (commit)
via 7bedb6e68dc97a4e081217c9a8901c31b7f9e13f (commit)
via 5912867d965664504148f12cff801836a56f0162 (commit)
via 9c5d1369b762e55838cc5aa24cdd61ebc079772d (commit)
via 30012ed996dd5336cbfa7394234d1cbbf08a2b78 (commit)
via 7d972883c71002a4ef11a7f85b47311b48847fa3 (commit)
via 10b25cacfc521b3dc74c2204fc2b29aca8ad2631 (commit)
via 42c4ef158ac51f01f071a285e4a131acbf753ab9 (commit)
via deaf93e06ddfb5346f0492f99f441aa4734e6bf5 (commit)
via 9259c169b6254ea581fcdcb18e1cdbe9b9fbea1e (commit)
via 8e17ce539b4a90711f4a8059a0eff11b9c64207b (commit)
via 4b8da3c9e3a48820ae3b1538103eda0282118392 (commit)
via bb7be2d3e49876a74fddb76260f3eac07bf0a431 (commit)
via 7fc67e3f9bfd12058e6f3d86d995704ed8962a8b (commit)
via a1379fdd1f825c22a3ada0a2c085a1b0121ce89a (commit)
via 829fd7e3d772182d322682505df2c99df4b2a9e2 (commit)
via 9b914107504ece419ee2f7d72be7d6262037ff52 (commit)
via 03980b49e2fb3cac357e417acea64cd342d1065e (commit)
via f2afe64c25d1b3e79b38972e69104431a2587935 (commit)
via cf2d5c1b966cae7d5023ff482891c4ea7e767de4 (commit)
via a9359cd6ff10866acc943889ee77ab3f971dfa4e (commit)
via 51a185e0f13ec3b2c376d3229c56d012dc91a04a (commit)
via 51e6fbab8d08a5a92fc08c552f841a84bccbe0d6 (commit)
via 8c1005ef92cffff5a4a9321f3b62da715b141c3d (commit)
via f71873ca6ec9969a90242a1c4d391770cabce1c2 (commit)
via ef8f43b830273dd54ae75f6a3823a946a455c2b3 (commit)
via 1ca973446ddf0094cd563a7af63f4e3078f5b772 (commit)
via a8b431b5cfccd36995514560f965b4943ac93c6b (commit)
via 36b1667a9b2ab888a80e84b6ecd75403c8f6782c (commit)
via 2a6a3720271a12bc8a47928ade5b2c641514ca44 (commit)
via d78a90009ca25da4f8cfa75672df0b5bd77cabfe (commit)
via 54819dd75fb929acfab581890dd1c8ee17cccf3e (commit)
via 2656de63a9531f23aea840f6cb92e0cdc803599f (commit)
via 2c9b40bdc8acd7fdd906c80e2c7f44abfb720de3 (commit)
via 19a2e9a97939126293ce33d72f576f6f54da574f (commit)
via e513251f7b0f6acdc0c0d6df5792c18358030221 (commit)
via 6feefc59c459d778046a83cb29178afc04acdf1c (commit)
via b95518608653185f96f378ea3df4cf1ad7b05817 (commit)
via 44494089c502572ee231bb421da70889b68fae4f (commit)
via 1de4f2fbc12c840726bc340792a73814818f3faa (commit)
via b20aa99b4356a2afa092bb355a0db78b0b8f711f (commit)
via 647c3bac741b56ce16c7e22ab2d462725a34198d (commit)
via 96b2b2fef69eaad2da73c1c5a0ee01f939089e15 (commit)
via 5467329d6822455de4644a277f741068cf5f1ec9 (commit)
via 0600e45775658866f624b87efeef6a1067db5c39 (commit)
via 8552d32092e45a1f6ee1424e92882ec84b51cb8a (commit)
via 0fde046a6c68909ae25af809557fcd64eb7264d7 (commit)
via 45a172b5f59ea7464b7241212464bf9113a18f36 (commit)
via c46ab7c622127315b4e90c98e859588a3403267a (commit)
via 381951637e6688e1868fae9b0642411f5cd1c223 (commit)
via 65f76d01b775c944d2ef6c1673633adb6229d9d8 (commit)
via 21f6342de550d985db7e2d6964af2ad0f8405406 (commit)
via 3c068341659783d0d7132f92b14c05c07830d5b2 (commit)
via a25971cae157b6bc40037e391db226da36dc9b30 (commit)
via 446868283108deda5510f4419b4065d254a4c5a8 (commit)
via 3987bcc3f5f6acf9207e94049561c632b454f38a (commit)
via d5341150545efd0960acf34186ca18b98a1b1860 (commit)
via 42e27f90df96881d22e365af7069f36c4538cf07 (commit)
via 749b308236ab70cd15088fcb81d093c9c5e8d30b (commit)
via 636d833ed4206ffe44e5e9b9b9100ac9d82bc603 (commit)
via 08a10afd0681a04e5bae019870d636d49c3dc223 (commit)
via a0daf49f4231d5b4ec6a8a00bfa0aac1ce69cde8 (commit)
via ca35c58c19b28dcdbbff71fe3da0ee16ca760673 (commit)
via 07f2ff368aa089e4f128578c49287d40d84dfcdf (commit)
via 687a5246e161ef1d191264d3b579271bc834223c (commit)
via ca9cbfd8d91e1ba968832ba98c656e32f59fc393 (commit)
from ffe6fa9c27060173d8fe292df0bd670dfd6fea8f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5c6c6946b034b821446a3657ca065b5646e0f104
Author: radhika <radhika at curoverse.com>
Date: Tue Sep 1 17:21:15 2015 -0400
6260: status check added; still not working completely since invoking datamanager singlerun issues.
diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index 4f9b149..492d402 100644
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -1,11 +1,13 @@
package main
import (
+ "encoding/json"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"git.curoverse.com/arvados.git/services/datamanager/keep"
+ "io"
"io/ioutil"
"log"
"net/http"
@@ -146,30 +148,43 @@ func DataManagerSingleRun(t *testing.T) {
}
}
-func MakeRequest(t *testing.T, path string) string {
+func MakeRequest(t *testing.T, path string) io.Reader {
client := http.Client{}
req, err := http.NewRequest("GET", path, strings.NewReader("resp"))
req.Header.Add("Authorization", "OAuth2 "+keep.GetDataManagerToken(nil))
req.Header.Add("Content-Type", "application/octet-stream")
resp, err := client.Do(req)
- if err != nil {
- t.Fatalf("Error during %s %s", path, err)
- }
+ defer resp.Body.Close()
- body, err := ioutil.ReadAll(resp.Body)
if err != nil {
- t.Fatalf("Error reading response from %s %s", path, err)
+ t.Fatalf("Error during %s %s", path, err)
}
- return string(body)
+ return resp.Body
}
func GetBlockIndexes(t *testing.T) []string {
var indexes []string
for i := 0; i < len(keepServers); i++ {
- resp := MakeRequest(t, keepServers[i]+"/index")
- lines := strings.Split(resp, "\n")
+ path := keepServers[i] + "/index"
+ client := http.Client{}
+ req, err := http.NewRequest("GET", path, strings.NewReader("resp"))
+ req.Header.Add("Authorization", "OAuth2 "+keep.GetDataManagerToken(nil))
+ req.Header.Add("Content-Type", "application/octet-stream")
+ resp, err := client.Do(req)
+ defer resp.Body.Close()
+
+ if err != nil {
+ t.Fatalf("Error during %s %s", path, err)
+ }
+
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ t.Fatalf("Error reading response from %s %s", path, err)
+ }
+
+ lines := strings.Split(string(body), "\n")
for _, line := range lines {
indexes = append(indexes, strings.Split(line, " ")...)
}
@@ -258,11 +273,22 @@ func BackdateBlocks(t *testing.T, oldBlockLocators []string) {
}
}
-func GetStatus(t *testing.T) {
- for i := 0; i < len(keepServers); i++ {
- resp := MakeRequest(t, keepServers[i]+"/status.json")
- log.Printf("Status from keepserver %d = %s", i, resp)
+func GetStatus(t *testing.T, path string) interface{} {
+ client := http.Client{}
+ req, err := http.NewRequest("GET", path, strings.NewReader("resp"))
+ req.Header.Add("Authorization", "OAuth2 "+keep.GetDataManagerToken(nil))
+ req.Header.Add("Content-Type", "application/octet-stream")
+ resp, err := client.Do(req)
+ defer resp.Body.Close()
+
+ if err != nil {
+ t.Fatalf("Error during %s %s", path, err)
}
+
+ var s interface{}
+ json.NewDecoder(resp.Body).Decode(&s)
+
+ return s
}
func TestPutAndGetBlocks(t *testing.T) {
@@ -302,42 +328,58 @@ func TestPutAndGetBlocks(t *testing.T) {
t.Fatalf("Locator of the collection with the same data as old block is different %s", old_block_collection_locator)
}
- GetStatus(t)
+ // Invoking datamanager singlerun or /index several times is resulting in errors
+ // Hence, for now just invoke once at the end of test
- /*
- // Invoking datamanager singlerun or /index several times is resulting in errors
- // Hence, for now just invoke once at the end of test
- var expected []string
- expected = append(expected, oldBlockLocators...)
- expected = append(expected, newBlockLocators...)
- expected = append(expected, to_delete_collection_locator)
+ var expected []string
+ expected = append(expected, oldBlockLocators...)
+ expected = append(expected, newBlockLocators...)
+ expected = append(expected, to_delete_collection_locator)
- VerifyBlocks(t, nil, expected)
+ VerifyBlocks(t, nil, expected)
- // Run datamanager in singlerun mode
- DataManagerSingleRun(t)
- */
+ // Run datamanager in singlerun mode
+ DataManagerSingleRun(t)
// Change mtime on old blocks and delete the collection
DeleteCollection(t, to_delete_collection_uuid)
BackdateBlocks(t, oldBlockLocators)
+ // Run data manager
time.Sleep(1 * time.Second)
DataManagerSingleRun(t)
- // Give some time for pull worker and trash worker to finish
- time.Sleep(5 * time.Second)
+ // Wait until PullQueue and TrashQueue finish their work
+ for {
+ var done [2]bool
+ for i := 0; i < 2; i++ {
+ s := GetStatus(t, keepServers[i]+"/status.json")
+ var pullQueueStatus interface{}
+ pullQueueStatus = s.(map[string]interface{})["PullQueue"]
+ var trashQueueStatus interface{}
+ trashQueueStatus = s.(map[string]interface{})["TrashQueue"]
+ if pullQueueStatus.(map[string]interface{})["Queued"] == float64(0) &&
+ pullQueueStatus.(map[string]interface{})["InProgress"] == float64(0) &&
+ trashQueueStatus.(map[string]interface{})["Queued"] == float64(0) &&
+ trashQueueStatus.(map[string]interface{})["InProgress"] == float64(0) {
+ done[i] = true
+ }
+ }
+ if done[0] && done[1] {
+ break
+ } else {
+ time.Sleep(1 * time.Second)
+ }
+ }
// Get block indexes and verify that the deleted collection block is no longer found
var not_expected []string
not_expected = append(not_expected, oldBlockLocators...)
not_expected = append(not_expected, to_delete_collection_locator)
-
- GetStatus(t)
VerifyBlocks(t, not_expected, newBlockLocators)
}
-// Invoking datamanager singlerun several times results in errors.
+// Invoking datamanager singlerun several times resulting in errors.
// Until that issue is resolved, don't run this test in the meantime.
func x_TestInvokeDatamanagerSingleRunRepeatedly(t *testing.T) {
defer TearDownDataManagerTest(t)
commit 505be2eb9ac1f36a5e31b374451193bc1ab7e90e
Merge: 829fd7e bd4c542
Author: radhika <radhika at curoverse.com>
Date: Tue Sep 1 10:54:33 2015 -0400
Merge branch 'master' into 6260-test-datamanager
commit 829fd7e3d772182d322682505df2c99df4b2a9e2
Merge: ffe6fa9 f2afe64
Author: radhika <radhika at curoverse.com>
Date: Wed Aug 26 18:05:39 2015 -0400
Merge branch 'master' into 6260-test-datamanager
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list