[ARVADOS] updated: f5617be935a121dc339effcaafebb569109ebe5b
Git user
git at public.curoverse.com
Fri Apr 22 18:54:49 EDT 2016
Summary of changes:
build/run-build-packages-one-target.sh | 1 +
build/run-build-packages.sh | 6 +-
sdk/cli/bin/arv-run-pipeline-instance | 4 +-
sdk/cwl/setup.py | 2 +-
sdk/go/keepclient/collectionreader.go | 4 +
sdk/go/keepclient/collectionreader_test.go | 1 +
sdk/go/manifest/manifest.go | 2 +-
sdk/go/streamer/transfer.go | 4 +-
sdk/ruby/arvados.gemspec | 10 ++-
sdk/ruby/lib/arvados.rb | 4 +-
sdk/ruby/lib/arvados/collection.rb | 12 +--
sdk/ruby/lib/arvados/keep.rb | 8 +-
sdk/ruby/test/test_keep_manifest.rb | 15 ++++
services/crunch-run/crunchrun_test.go | 4 +-
services/datamanager/loggerutil/loggerutil.go | 2 +-
services/datamanager/summary/pull_list.go | 2 +-
services/fuse/tests/test_mount.py | 1 -
services/keep-web/handler.go | 6 ++
services/keepproxy/keepproxy.go | 2 +-
services/keepstore/trash_worker_test.go | 2 +-
.../arvnodeman/computenode/dispatch/__init__.py | 99 ++++++++++----------
.../arvnodeman/computenode/dispatch/slurm.py | 53 ++++-------
.../arvnodeman/computenode/dispatch/transitions.py | 52 +++++++++++
services/nodemanager/arvnodeman/daemon.py | 22 ++---
.../nodemanager/tests/test_computenode_dispatch.py | 37 ++++----
.../tests/test_computenode_dispatch_slurm.py | 100 +++++----------------
services/nodemanager/tests/test_daemon.py | 32 +------
services/nodemanager/tests/testutil.py | 7 ++
tools/arvbox/bin/arvbox | 3 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 2 +-
.../{service/api/run-service => api-setup.sh} | 18 +---
tools/arvbox/lib/arvbox/docker/common.sh | 1 +
.../lib/arvbox/docker/service/api/run-service | 78 +---------------
.../{workbench => websockets}/log/main/.gitstub | 0
.../service/{workbench => websockets}/log/run | 0
.../arvbox/docker/service/{sso => websockets}/run | 0
.../arvbox/docker/service/websockets/run-service | 30 +++++++
tools/keep-block-check/keep-block-check.go | 13 +--
tools/keep-block-check/keep-block-check_test.go | 15 +++-
tools/keep-rsync/keep-rsync.go | 23 ++---
tools/keep-rsync/keep-rsync_test.go | 4 +-
41 files changed, 304 insertions(+), 377 deletions(-)
create mode 100644 services/nodemanager/arvnodeman/computenode/dispatch/transitions.py
copy tools/arvbox/lib/arvbox/docker/{service/api/run-service => api-setup.sh} (83%)
copy tools/arvbox/lib/arvbox/docker/service/{workbench => websockets}/log/main/.gitstub (100%)
copy tools/arvbox/lib/arvbox/docker/service/{workbench => websockets}/log/run (100%)
copy tools/arvbox/lib/arvbox/docker/service/{sso => websockets}/run (100%)
create mode 100755 tools/arvbox/lib/arvbox/docker/service/websockets/run-service
via f5617be935a121dc339effcaafebb569109ebe5b (commit)
via 628f2f2e1bfabdb7221badab3d5189011aee5a54 (commit)
via 08d8c1af187132692baf70a531f87c8082f72cdc (commit)
via 3c88abd3cb33cbe80bb81a7cca779fe668036c9e (commit)
via 469102b30008b4ee73c9f44b03a6bd82a2a91fb2 (commit)
via 0cacef6ec2b553052e3897fee3067951f65117cd (commit)
via f2e0e32c7ab0f1de29e6e90a6af87c1969dfd640 (commit)
via b2cfb1a8873ee8c4128bd3bdd9fb32b9d9e8ee6e (commit)
via d7b6ec65cfd774f0d17c96d2baf9b6620c55ad8d (commit)
via 2617eb303578ff5fdf9f24fd4a5de4bfd2144425 (commit)
via caa404bdf14b29dc592d5b8b87fc264c1cbce8ae (commit)
via 9b90fe97f8e92794856e3d730811953e1c13ea25 (commit)
via 611244398282b805188d4c334f3323807ca394c5 (commit)
via 29379beaa615d5a36032a05e71d7a2730e255c48 (commit)
via 6f1543559341624f0b9bdb880f7feb3fb8495811 (commit)
via aaab24cd029c049963acf74ff8ec3694c599ba20 (commit)
via b51781b96a666ac5e671dfed8c255ac02f7da163 (commit)
via c213eb1a85c5a0cb26000dc1d013cb11c0f8c601 (commit)
via 827f01731858dc21578f0bd35f01d9d58b72d2ef (commit)
via fe8143ccd8193e4d7d2098d0a61dce7127aef713 (commit)
via dfdb6060ea1a6544fe2497d7c1d18078879fe090 (commit)
via 17c23d338518f0498fb1396f24954f884a06b05b (commit)
via 8704a73054ecf639ac12cd4de4b1349ed69ae093 (commit)
via da39f7ee464b4be805591ff5901bdacaa005b0a8 (commit)
via da6e931b990c7edde55213196fc8a47f4dc4998d (commit)
via e7091ec45f8f634a52db2fba1b385e790d69e6fe (commit)
via 3dc4c3e304a1d2ccb975a43679276eed55bd49fd (commit)
via 51f64605f9b033efcc02b45376f2b7f22e360483 (commit)
via abfdd429351ad6b457667b624c8ba550ffcac999 (commit)
via 2a13fcc84b240e368787c8e94ced95d75eee0cc4 (commit)
via 26751323e77005dc158b64e86c47bbb9459e6697 (commit)
via fd696fac8234ff799aa6ac63819e5e6ac0a8a9d4 (commit)
via 3a31612fdc5e1ccdfa91d04ff5a089d6e06c0f60 (commit)
via 065cefa6e87c8b668af5a4abc065514b84db83b5 (commit)
via 40ad5fb3f699c348222c90692b9ae27a8f716af9 (commit)
from 83332c0e546ad5e6332ce507224d9c6f64552ce6 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f5617be935a121dc339effcaafebb569109ebe5b
Author: radhika <radhika at curoverse.com>
Date: Fri Apr 22 18:51:17 2016 -0400
8936: update keep-block-check and keep-rsync to properly use blob-signature-ttl to perform the signature verification.
diff --git a/tools/keep-block-check/keep-block-check.go b/tools/keep-block-check/keep-block-check.go
index 569046b..6cf11a7 100644
--- a/tools/keep-block-check/keep-block-check.go
+++ b/tools/keep-block-check/keep-block-check.go
@@ -48,7 +48,7 @@ func doMain(args []string) error {
"",
"Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.")
- blobSignatureTTL := flags.Duration(
+ blobSignatureTTLFlag := flags.Duration(
"blob-signature-ttl",
0,
"Lifetime of blob permission signatures on the keepservers. If not provided, this will be retrieved from the API server's discovery document.")
@@ -73,12 +73,12 @@ func doMain(args []string) error {
}
// setup keepclient
- kc, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTL)
+ kc, blobSignatureTTL, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTLFlag)
if err != nil {
return fmt.Errorf("Error configuring keepclient: %s", err.Error())
}
- return performKeepBlockCheck(kc, *blobSignatureTTL, blobSigningKey, blockLocators, *verbose)
+ return performKeepBlockCheck(kc, blobSignatureTTL, blobSigningKey, blockLocators, *verbose)
}
type apiConfig struct {
@@ -143,7 +143,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin
}
// setup keepclient using the config provided
-func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, err error) {
+func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, ttl time.Duration, err error) {
arv := arvadosclient.ArvadosClient{
ApiToken: config.APIToken,
ApiServer: config.APIHost,
@@ -168,12 +168,13 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL
}
// Get if blobSignatureTTL is not provided
+ ttl = blobSignatureTTL
if blobSignatureTTL == 0 {
value, err := arv.Discovery("blobSignatureTtl")
if err == nil {
- blobSignatureTTL = time.Duration(int(value.(float64))) * time.Second
+ ttl = time.Duration(int(value.(float64))) * time.Second
} else {
- return nil, err
+ return nil, 0, err
}
}
diff --git a/tools/keep-block-check/keep-block-check_test.go b/tools/keep-block-check/keep-block-check_test.go
index 6c77612..e49fe68 100644
--- a/tools/keep-block-check/keep-block-check_test.go
+++ b/tools/keep-block-check/keep-block-check_test.go
@@ -72,6 +72,10 @@ func (s *DoMainTestSuite) TearDownTest(c *C) {
}
func setupKeepBlockCheck(c *C, enforcePermissions bool, keepServicesJSON string) {
+ setupKeepBlockCheckWithTTL(c, enforcePermissions, keepServicesJSON, blobSignatureTTL)
+}
+
+func setupKeepBlockCheckWithTTL(c *C, enforcePermissions bool, keepServicesJSON string, ttl time.Duration) {
var config apiConfig
config.APIHost = os.Getenv("ARVADOS_API_HOST")
config.APIToken = arvadostest.DataManagerToken
@@ -82,7 +86,8 @@ func setupKeepBlockCheck(c *C, enforcePermissions bool, keepServicesJSON string)
// setup keepclients
var err error
- kc, err = setupKeepClient(config, keepServicesJSON, blobSignatureTTL)
+ kc, ttl, err = setupKeepClient(config, keepServicesJSON, ttl)
+ c.Assert(ttl, Equals, blobSignatureTTL)
c.Check(err, IsNil)
}
@@ -167,6 +172,14 @@ func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) {
checkNoErrorsLogged(c, "Error verifying block", "Block not found")
}
+func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigningAndTTLFromDiscovery(c *C) {
+ setupKeepBlockCheckWithTTL(c, true, "", 0)
+ allLocators := setupTestData(c)
+ err := performKeepBlockCheck(kc, blobSignatureTTL, arvadostest.BlobSigningKey, allLocators, true)
+ c.Check(err, IsNil)
+ checkNoErrorsLogged(c, "Error verifying block", "Block not found")
+}
+
func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) {
setupKeepBlockCheck(c, false, "")
allLocators := setupTestData(c)
diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index 36f7320..c6e7665 100644
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -60,7 +60,7 @@ func doMain() error {
"",
"Index prefix")
- srcBlobSignatureTTL := flags.Duration(
+ srcBlobSignatureTTLFlag := flags.Duration(
"src-blob-signature-ttl",
0,
"Lifetime of blob permission signatures on source keepservers. If not provided, this will be retrieved from the API server's discovery document.")
@@ -79,18 +79,18 @@ func doMain() error {
}
// setup src and dst keepclients
- kcSrc, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0, *srcBlobSignatureTTL)
+ kcSrc, srcBlobSignatureTTL, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0, *srcBlobSignatureTTLFlag)
if err != nil {
return fmt.Errorf("Error configuring src keepclient: %s", err.Error())
}
- kcDst, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications, 0)
+ kcDst, _, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications, 0)
if err != nil {
return fmt.Errorf("Error configuring dst keepclient: %s", err.Error())
}
// Copy blocks not found in dst from src
- err = performKeepRsync(kcSrc, kcDst, *srcBlobSignatureTTL, srcBlobSigningKey, *prefix)
+ err = performKeepRsync(kcSrc, kcDst, srcBlobSignatureTTL, srcBlobSigningKey, *prefix)
if err != nil {
return fmt.Errorf("Error while syncing data: %s", err.Error())
}
@@ -160,7 +160,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin
}
// setup keepclient using the config provided
-func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int, srcBlobSignatureTTL time.Duration) (kc *keepclient.KeepClient, err error) {
+func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int, srcBlobSignatureTTL time.Duration) (kc *keepclient.KeepClient, blobSignatureTTL time.Duration, err error) {
arv := arvadosclient.ArvadosClient{
ApiToken: config.APIToken,
ApiServer: config.APIHost,
@@ -174,13 +174,13 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl
if keepServicesJSON == "" {
kc, err = keepclient.MakeKeepClient(&arv)
if err != nil {
- return nil, err
+ return nil, 0, err
}
} else {
kc = keepclient.New(&arv)
err = kc.LoadKeepServicesFromJSON(keepServicesJSON)
if err != nil {
- return kc, err
+ return kc, 0, err
}
}
@@ -191,7 +191,7 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl
if err == nil {
replications = int(value.(float64))
} else {
- return nil, err
+ return nil, 0, err
}
}
@@ -199,16 +199,17 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl
}
// If srcBlobSignatureTTL is not provided, get it from API server discovery doc
+ blobSignatureTTL = srcBlobSignatureTTL
if !isDst && srcBlobSignatureTTL == 0 {
value, err := arv.Discovery("blobSignatureTtl")
if err == nil {
- srcBlobSignatureTTL = time.Duration(int(value.(float64))) * time.Second
+ blobSignatureTTL = time.Duration(int(value.(float64))) * time.Second
} else {
- return nil, err
+ return nil, 0, err
}
}
- return kc, nil
+ return kc, blobSignatureTTL, nil
}
// Get unique block locators from src and dst
diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index f1f6a5f..bdae39a 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -100,10 +100,10 @@ func setupRsync(c *C, enforcePermissions bool, replications int) {
// setup keepclients
var err error
- kcSrc, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL)
+ kcSrc, _, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL)
c.Check(err, IsNil)
- kcDst, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0)
+ kcDst, _, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0)
c.Check(err, IsNil)
for uuid := range kcSrc.LocalRoots() {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list