[ARVADOS] updated: ca7460bcc97a8b1402b81ecebc3f0ac3131230c0
git at public.curoverse.com
git at public.curoverse.com
Tue Mar 3 12:27:10 EST 2015
Summary of changes:
crunch_scripts/split-fastq.py | 72 +-
...nstall-manual-prerequisites.html.textile.liquid | 3 +-
doc/install/install-sso.html.textile.liquid | 36 +-
sdk/python/arvados/__init__.py | 19 +-
sdk/python/arvados/_normalize_stream.py | 55 +
sdk/python/arvados/_ranges.py | 204 +++
sdk/python/arvados/api.py | 59 +-
sdk/python/arvados/arvfile.py | 900 ++++++++++++-
sdk/python/arvados/collection.py | 1320 ++++++++++++++++----
sdk/python/arvados/commands/arv_copy.py | 109 +-
sdk/python/arvados/commands/put.py | 4 +-
sdk/python/arvados/config.py | 7 +-
sdk/python/arvados/errors.py | 47 +-
sdk/python/arvados/keep.py | 129 +-
sdk/python/arvados/safeapi.py | 31 +
sdk/python/arvados/stream.py | 274 +---
sdk/python/tests/arvados_testutil.py | 34 +-
sdk/python/tests/run_test_server.py | 2 +-
sdk/python/tests/test_arv_put.py | 42 +-
sdk/python/tests/test_arvfile.py | 588 +++++++++
sdk/python/tests/test_collections.py | 560 +++++++--
sdk/python/tests/test_errors.py | 30 +-
sdk/python/tests/test_keep_client.py | 56 +-
sdk/python/tests/test_keep_locator.py | 6 +-
sdk/python/tests/test_sdk.py | 12 +-
sdk/python/tests/test_stream.py | 25 +-
services/arv-web/arv-web.py | 5 +-
services/fuse/arvados_fuse/__init__.py | 38 +-
services/fuse/bin/arv-mount | 3 +-
services/fuse/setup.py | 2 +-
services/fuse/tests/test_mount.py | 3 +-
services/keepstore/keepstore.go | 14 +-
services/keepstore/pull_worker.go | 16 +-
services/keepstore/pull_worker_test.go | 8 +-
.../arvnodeman/computenode/driver/gce.py | 12 -
services/nodemanager/doc/gce.example.cfg | 7 +-
.../tests/test_computenode_driver_gce.py | 15 -
37 files changed, 3756 insertions(+), 991 deletions(-)
create mode 100644 sdk/python/arvados/_normalize_stream.py
create mode 100644 sdk/python/arvados/_ranges.py
create mode 100644 sdk/python/arvados/safeapi.py
create mode 100644 sdk/python/tests/test_arvfile.py
via ca7460bcc97a8b1402b81ecebc3f0ac3131230c0 (commit)
via 8ba8aece3e72247d5dbf885b5845a1663956032d (commit)
via 7e814093e36dd6fc96353584c2c0797f4654f317 (commit)
via 6cd3b6fb1c143017c6bca4db899c8c6ddd237572 (commit)
via 2365c0eb0110a56be7a6d8046b499649a1e78148 (commit)
via a380fd23cdfafa399327dfe666c1ef8e94804779 (commit)
via 1d45536fa815efe3cb2eba4d00fb4d368b5dfbca (commit)
via 83562734c781379665e3a337bce5b548afc88681 (commit)
via b887df487c6cb465c3212e8ef99278350e58ff0a (commit)
via b3bccdb71967ea963e9a747a3521bd99e33e12a4 (commit)
via 116d6fd671641ecb7d1d491ef4798a0d1b5b48ac (commit)
via e2ab0a122f3b721a7663b61769f15713dbcf1a5e (commit)
via d3b57b6111e49f80c725bb85b0c7883008884309 (commit)
via 74b568d3b0ae82f853618e49c128c1258630b125 (commit)
via bb19e060336c4e734b3e1922c5be3c4b40ff7da8 (commit)
via 256068ede1b17e74288d645d3c23fea494036f47 (commit)
via c691e44f9183176581dd8da6417af16772baf510 (commit)
via ae41687970e380effecc588301069b604e0ba041 (commit)
via d0ea98ecf4c625928591ed9823fafef2b9da5101 (commit)
via c3be77dc1329ac775c686fd40ca1014a799a3a79 (commit)
via 63b03a39adfd78961c5bbb6a3a2d02ccd8c92e4d (commit)
via cb8320961335a4732c470882fcfcd4e6c581a0d4 (commit)
via 0c9dd4b566696e5de7bbe82d000997acc978dcfa (commit)
via a1fc5b9e889f8359a32470c3a7d91190d0894899 (commit)
via ad69b48e324c3ce29a4d2c84732dfd3d0288ebb3 (commit)
via 2548543949ac90b51d555607b115d0e0efd45640 (commit)
via 3070d89e1e52cbc42a04560f5db2630c0c4cbd6b (commit)
via 93d38f3cb3d7874ed3723409357d8dd5e8d618a5 (commit)
via 62e3f155b14b30fde1323a367351c70b23f312b8 (commit)
via f39118dacff5614e1102bb18f6dc850aecf2186e (commit)
via 0d63ac0c2486a43198eb1015ba8d1028239139ee (commit)
via 27fe76f518db97633148657ff1e1610095be11fa (commit)
via 50d3be922d18357f1033b02c7add035fba4acd56 (commit)
via 20b2d6f7560e82add928fa14d868c9a4319d4907 (commit)
via 503c616f17ec4b77ce1778cc2033e6ed78927f0d (commit)
via 5f905666581dd7ccd8f7e05d1c8c4a6eedff0da9 (commit)
via 446785a11237f832e9bac540a04e7dd238b45a9e (commit)
via f22dda84271714c924c322bafba5e9f184e4fb33 (commit)
via f8f654db2a5efc85a358dfd126be4ccb1abb322d (commit)
via 8ae4b9342772ce4693b5961da5e83f54a36ad0e7 (commit)
via f6b88a9e7e9727f0397f1febbad8c08f6a20463d (commit)
via 8128b451b8e3161ae3db8b7e1858f6eef8105c9f (commit)
via cef69b16e1295e456548813ad95c9db827136c17 (commit)
via 08c3828aa71b5967707208a9fbe4a70198ff8279 (commit)
via 19f7fd5eac2f65414eab857a17dfe727501bb294 (commit)
via da63a23ebc1d936305324cbc66d9903b9129df94 (commit)
via fef791011864b1bd69dfd01bfcd26c4942d6d5a1 (commit)
via 66f42779127b7e48ef4437b7d4ec471e43f9636b (commit)
via bac1bf1fa5154b0db653d5ba0353458e1fc24dd2 (commit)
via 5c0430fd80db2fd3d57243d9187588c6854a738b (commit)
via d42850766752bad0f1c2be55534b80617bcda411 (commit)
via c30d3a7cc1c647ed5339ac15c8d8df07a76c712d (commit)
via 130b8d3147af5240ba3118c56b0aa1f19d2d5c59 (commit)
via a7b16675337995c65b909a7519178efed98a3089 (commit)
via 8192c879caaca60902ca362b4967cf4492d08e0c (commit)
via f56d3a6876f246f78d5bc231a0ac5b6e4c6bdb9c (commit)
via 070320475e3379be2d6c79bfd581d08e50274d33 (commit)
via 7dcac8234708b244251f957a4fc2ceb68ec06881 (commit)
via 951149b9258803cb46162ceb78bbec0afae50a04 (commit)
via 438752b8a79d58a615e51bd5ddcbea74b1452a63 (commit)
via 5bb11227e0d7d37cc6cf574b5d5f289afd1a43b1 (commit)
via facbf48de90f600a42363b72bcb2b4929f78c73d (commit)
via 2b60134e4a943dc4cf76f9ff81106ad2ce695bfa (commit)
via 58d12cd824d82bd8c47f6d9c4cb1b0509b994950 (commit)
via 276ca380e9ee106b3880708d0ac48e2ab3cca335 (commit)
via 616a9b2eb14e49d8f38ef5d72bd875378450d35f (commit)
via c78e31f6dd65f215bb349ae2d7bd7d59fca1076c (commit)
via 33606c82797185fa65751f5724304f02605e0e05 (commit)
via 04f5f75659e963f02441a566afcc6d22683d0485 (commit)
via 86f94e641462ddd78a454fc0a0cdc9dd4c4d9bef (commit)
via 0e9443678fc7fd4862b76a10184dc7357a75155d (commit)
via 89796f01a6ea3cb553a61be6ce92883a1decf003 (commit)
via 22286e8b81fa7644500e197b95e6d6417ed25f7e (commit)
via cdf4cf8189b0a56c25f63d39a1dc916082d08ecd (commit)
via d251327b574e981cc02ff04c80d609df5c9ec30d (commit)
via 20f65ff9a7c6c6f73b152d122b70fb44ea5a21a4 (commit)
via 68cd78ca1a20d80e0ab90d125df305f30b606f85 (commit)
via 5109db362064053ed6711169d6c414b2cb4e22fb (commit)
via 02e9754a68a5816458d517b8f5012530cf17ebba (commit)
via 88ba4c97290a316568d89985636d7d2b8fbcfa92 (commit)
via a117d9790fc566653aead700f8cb207858c8fca9 (commit)
via 1811fb602be08a1f9ff9f71070861d8a2af60849 (commit)
via b3cdfeea8341ac3b22ec84d58645523a59eceba9 (commit)
via e78607008c39aa88ccf6e95d7c6dfcc20a52a2ed (commit)
via 93bc7c31775039cfb05be4caa0891f13fa49409f (commit)
via ad638d5e0e28ab260df272a33bdeaa2b56a96054 (commit)
via 7abed45cc9b1c5e816584b9f267f6c448384ffed (commit)
via cc946c07eade09974423955f8d4b080941f53c7b (commit)
via 96d2964dcbc02c3873084ee09183f0f5fb0c44ba (commit)
via b5efdd0afbe7795b036dc19b8f7d6b8a32da52df (commit)
via b0e2fe9d0a18d5e809bc8c0d3382e0e023cb949c (commit)
via 71959d0d8434b7af5de9a5b8260f0ebe11ec7238 (commit)
via 78700dbcc4f5d34f0a4cfee5c040e716d684ed62 (commit)
via 5bf8784175517327b334fb4f7c8d3b8ee505353d (commit)
via fef134f2bab9d20e79e265e78de9ec83131e9f90 (commit)
from ba032ff921ffb7be57d9f831e2a3dce8b38e266e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ca7460bcc97a8b1402b81ecebc3f0ac3131230c0
Merge: 8ba8aec 7e81409
Author: Radhika Chippada <radhika at curoverse.com>
Date: Tue Mar 3 12:26:59 2015 -0500
Merge branch 'master' into 3761-pull-list-worker
commit 8ba8aece3e72247d5dbf885b5845a1663956032d
Author: Radhika Chippada <radhika at curoverse.com>
Date: Tue Mar 3 12:26:21 2015 -0500
3761: pass keepclient as an arg to RunPullWorker
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index b6ab8fa..ca79667 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -13,6 +13,8 @@ import (
"strings"
"syscall"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
)
// ======================
@@ -275,8 +277,18 @@ func main() {
}
// Initialize Pull queue and worker
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ log.Fatalf("Error setting up arvados client %s", err.Error())
+ }
+
+ keepClient, err := keepclient.MakeKeepClient(&arv)
+ if err != nil {
+ log.Fatalf("Error setting up keep client %s", err.Error())
+ }
+
pullq = NewWorkQueue()
- go RunPullWorker(pullq.NextItem)
+ go RunPullWorker(pullq.NextItem, keepClient)
// Shut down the server gracefully (by closing the listener)
// if SIGTERM is received.
diff --git a/services/keepstore/pull_worker.go b/services/keepstore/pull_worker.go
index db2b54d..dced531 100644
--- a/services/keepstore/pull_worker.go
+++ b/services/keepstore/pull_worker.go
@@ -4,7 +4,6 @@ import (
"crypto/rand"
"errors"
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"io"
"io/ioutil"
@@ -13,7 +12,6 @@ import (
"time"
)
-var arv arvadosclient.ArvadosClient
var keepClient keepclient.KeepClient
/*
@@ -24,18 +22,8 @@ var keepClient keepclient.KeepClient
Skip the rest of the servers if no errors
Repeat
*/
-func RunPullWorker(nextItem <-chan interface{}) {
- var err error
- arv, err = arvadosclient.MakeArvadosClient()
- if err != nil {
- log.Fatalf("Error setting up arvados client %s", err.Error())
- }
-
- keepClient, err = keepclient.MakeKeepClient(&arv)
- if err != nil {
- log.Fatalf("Error setting up keep client %s", err.Error())
- }
-
+func RunPullWorker(nextItem <-chan interface{}, kc keepclient.KeepClient) {
+ keepClient = kc
for item := range nextItem {
Pull(item.(PullRequest))
}
diff --git a/services/keepstore/pull_worker_test.go b/services/keepstore/pull_worker_test.go
index de9ec41..84e951c 100644
--- a/services/keepstore/pull_worker_test.go
+++ b/services/keepstore/pull_worker_test.go
@@ -3,6 +3,8 @@ package main
import (
"bytes"
"errors"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
. "gopkg.in/check.v1"
"io"
"net/http"
@@ -27,7 +29,11 @@ var _ = Suite(&PullWorkerTestSuite{})
func (s *PullWorkerTestSuite) SetUpSuite(c *C) {
// Since keepstore does not come into picture in tests,
// we need to explicitly start the goroutine in tests.
- go RunPullWorker(pullq.NextItem)
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, Equals, nil)
+ keepClient, err := keepclient.MakeKeepClient(&arv)
+ c.Assert(err, Equals, nil)
+ go RunPullWorker(pullq.NextItem, keepClient)
// When a new pull request arrives, the old one will be overwritten.
// This behavior is simulated with delay tests below.
commit b887df487c6cb465c3212e8ef99278350e58ff0a
Merge: ba032ff 116d6fd
Author: Radhika Chippada <radhika at curoverse.com>
Date: Tue Mar 3 11:05:53 2015 -0500
Merge branch 'master' into 3761-pull-list-worker
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list