[ARVADOS] updated: 49fa621b6ea36479a0581df9f258ad032bc046d3
git at public.curoverse.com
git at public.curoverse.com
Wed Dec 16 12:43:45 EST 2015
Summary of changes:
.gitignore | 7 +-
.../controllers/container_requests_controller.rb | 2 +
.../app/controllers/containers_controller.rb | 2 +
apps/workbench/app/models/container.rb | 3 +
apps/workbench/app/models/container_request.rb | 3 +
apps/workbench/app/models/pipeline_instance.rb | 4 +
.../pipeline_instances/_running_component.html.erb | 29 +-
.../_show_components_running.html.erb | 2 +-
.../views/pipeline_instances/_show_log.html.erb | 3 +-
apps/workbench/config/routes.rb | 2 +
.../test/integration/anonymous_access_test.rb | 19 +-
apps/workbench/test/integration/jobs_test.rb | 4 +-
.../test/integration/pipeline_instances_test.rb | 12 +-
apps/workbench/test/integration_helper.rb | 2 +-
LICENSE-2.0.txt => sdk/cli/LICENSE-2.0.txt | 0
sdk/cli/arvados-cli.gemspec | 4 +-
sdk/go/arvadostest/fixtures.go | 9 +
sdk/go/keepclient/discover.go | 4 +-
sdk/go/keepclient/keepclient.go | 3 +-
sdk/go/keepclient/keepclient_test.go | 37 +-
sdk/go/keepclient/support.go | 15 +-
LICENSE-2.0.txt => sdk/pam/LICENSE-2.0.txt | 0
sdk/pam/MANIFEST.in | 1 +
sdk/pam/setup.py | 4 +-
LICENSE-2.0.txt => sdk/python/LICENSE-2.0.txt | 0
sdk/python/MANIFEST.in | 1 +
sdk/python/arvados/commands/run.py | 2 +-
sdk/python/arvados/keep.py | 12 +-
sdk/python/setup.py | 3 +
LICENSE-2.0.txt => sdk/ruby/LICENSE-2.0.txt | 0
sdk/ruby/arvados.gemspec | 3 +-
.../arvados/v1/container_requests_controller.rb | 6 +
.../arvados/v1/containers_controller.rb | 7 +
services/api/app/models/container.rb | 171 ++++++++++
services/api/app/models/container_request.rb | 175 ++++++++++
services/api/config/routes.rb | 2 +
...0151202151426_create_containers_and_requests.rb | 59 ++++
.../migrate/20151215134304_fix_containers_index.rb | 17 +
services/api/db/structure.sql | 186 ++++++++++-
services/api/lib/whitelist_update.rb | 18 +
services/api/test/fixtures/jobs.yml | 2 +-
services/api/test/unit/container_request_test.rb | 371 +++++++++++++++++++++
services/api/test/unit/container_test.rb | 200 +++++++++++
services/datamanager/collection/collection.go | 24 +-
services/datamanager/collection/collection_test.go | 6 +-
services/datamanager/datamanager.go | 34 +-
services/datamanager/summary/file.go | 49 ++-
services/dockercleaner/MANIFEST.in | 1 +
.../dockercleaner/agpl-3.0.txt | 0
services/dockercleaner/setup.py | 3 +
services/fuse/MANIFEST.in | 1 +
agpl-3.0.txt => services/fuse/agpl-3.0.txt | 0
services/fuse/arvados_fuse/command.py | 20 +-
services/fuse/setup.py | 3 +
services/fuse/tests/test_command_args.py | 75 +++++
services/keepstore/azure_blob_volume.go | 5 +-
services/keepstore/azure_blob_volume_test.go | 26 +-
services/keepstore/bufferpool_test.go | 6 -
services/keepstore/collision_test.go | 6 -
services/keepstore/gocheck_test.go | 10 +
.../keepstore/handlers_with_generic_volume_test.go | 15 +-
services/keepstore/keepstore_test.go | 8 +-
services/keepstore/pull_worker_test.go | 11 +-
services/keepstore/s3_volume.go | 312 +++++++++++++++++
services/keepstore/s3_volume_test.go | 133 ++++++++
services/keepstore/volume_generic_test.go | 76 +++--
services/keepstore/volume_unix_test.go | 12 +-
agpl-3.0.txt => services/login-sync/agpl-3.0.txt | 0
services/login-sync/arvados-login-sync.gemspec | 2 +-
services/nodemanager/MANIFEST.in | 1 +
agpl-3.0.txt => services/nodemanager/agpl-3.0.txt | 0
.../arvnodeman/computenode/driver/gce.py | 14 +-
services/nodemanager/arvnodeman/config.py | 10 +-
services/nodemanager/setup.py | 5 +-
.../tests/test_computenode_driver_gce.py | 30 ++
services/nodemanager/tests/test_config.py | 2 +
tools/crunchstat-summary/.gitignore | 2 +
tools/crunchstat-summary/MANIFEST.in | 1 +
.../crunchstat-summary/agpl-3.0.txt | 0
tools/crunchstat-summary/bin/crunchstat-summary | 12 +
.../crunchstat_summary}/__init__.py | 0
.../crunchstat_summary/command.py | 14 +
.../crunchstat_summary/summarizer.py | 89 +++++
.../crunchstat-summary}/gittaggers.py | 0
.../fuse => tools/crunchstat-summary}/setup.py | 23 +-
.../crunchstat-summary/tests}/__init__.py | 0
.../tests/logfile_20151204190335.txt.gz | Bin 0 -> 3639 bytes
.../tests/logfile_20151204190335.txt.gz.report | 20 ++
.../tests/logfile_20151210063411.txt.gz | Bin 0 -> 3093 bytes
.../tests/logfile_20151210063411.txt.gz.report | 10 +
.../tests/logfile_20151210063439.txt.gz | Bin 0 -> 3105 bytes
.../tests/logfile_20151210063439.txt.gz.report | 10 +
tools/crunchstat-summary/tests/test_examples.py | 22 ++
tools/keep-exercise/keep-exercise.go | 25 ++
94 files changed, 2276 insertions(+), 258 deletions(-)
create mode 100644 apps/workbench/app/controllers/container_requests_controller.rb
create mode 100644 apps/workbench/app/controllers/containers_controller.rb
create mode 100644 apps/workbench/app/models/container.rb
create mode 100644 apps/workbench/app/models/container_request.rb
copy LICENSE-2.0.txt => sdk/cli/LICENSE-2.0.txt (100%)
copy LICENSE-2.0.txt => sdk/pam/LICENSE-2.0.txt (100%)
copy LICENSE-2.0.txt => sdk/python/LICENSE-2.0.txt (100%)
copy LICENSE-2.0.txt => sdk/ruby/LICENSE-2.0.txt (100%)
create mode 100644 services/api/app/controllers/arvados/v1/container_requests_controller.rb
create mode 100644 services/api/app/controllers/arvados/v1/containers_controller.rb
create mode 100644 services/api/app/models/container.rb
create mode 100644 services/api/app/models/container_request.rb
create mode 100644 services/api/db/migrate/20151202151426_create_containers_and_requests.rb
create mode 100644 services/api/db/migrate/20151215134304_fix_containers_index.rb
create mode 100644 services/api/lib/whitelist_update.rb
create mode 100644 services/api/test/unit/container_request_test.rb
create mode 100644 services/api/test/unit/container_test.rb
create mode 100644 services/dockercleaner/MANIFEST.in
copy agpl-3.0.txt => services/dockercleaner/agpl-3.0.txt (100%)
copy agpl-3.0.txt => services/fuse/agpl-3.0.txt (100%)
create mode 100644 services/keepstore/gocheck_test.go
create mode 100644 services/keepstore/s3_volume.go
create mode 100644 services/keepstore/s3_volume_test.go
copy agpl-3.0.txt => services/login-sync/agpl-3.0.txt (100%)
copy agpl-3.0.txt => services/nodemanager/agpl-3.0.txt (100%)
create mode 100644 tools/crunchstat-summary/.gitignore
create mode 100644 tools/crunchstat-summary/MANIFEST.in
copy agpl-3.0.txt => tools/crunchstat-summary/agpl-3.0.txt (100%)
create mode 100755 tools/crunchstat-summary/bin/crunchstat-summary
copy {services/fuse/tests/performance => tools/crunchstat-summary/crunchstat_summary}/__init__.py (100%)
create mode 100644 tools/crunchstat-summary/crunchstat_summary/command.py
create mode 100644 tools/crunchstat-summary/crunchstat_summary/summarizer.py
copy {services/nodemanager => tools/crunchstat-summary}/gittaggers.py (100%)
copy {services/fuse => tools/crunchstat-summary}/setup.py (62%)
mode change 100644 => 100755
copy {services/fuse/tests/performance => tools/crunchstat-summary/tests}/__init__.py (100%)
create mode 100644 tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz
create mode 100644 tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz.report
create mode 100644 tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz
create mode 100644 tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz.report
create mode 100644 tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz
create mode 100644 tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz.report
create mode 100644 tools/crunchstat-summary/tests/test_examples.py
discards 537a84cd45c915ea7b6f607b17c1cf665ed71561 (commit)
via 49fa621b6ea36479a0581df9f258ad032bc046d3 (commit)
via 08ee1fa80cc4718bce28283c67954b70ef6d7267 (commit)
via d71335c6af0e312b451fdc99ecae42362ba4723d (commit)
via 637b32482c7fd0fc14197b120b3657793646216f (commit)
via 6248a5d1b8561dec7298e20e2858cc003df5a763 (commit)
via 242b1cc2ca6b85f7bb9e176a786d7c484eb35c53 (commit)
via 40158e8f7027d51ca704c3fd6039818acf2b21c0 (commit)
via 4e5342f47855cb76c22bdb9095c7e459701833bd (commit)
via e74e37c42a38fbd985f35d58796ae008194e9d23 (commit)
via a1adf1ed6f93ce0769f307a86b6389e9e8e630a9 (commit)
via 081e0ce2c669068f5684a5e0a30935294a90147f (commit)
via 85782d1376ba94714088499b064ba3e43aefe722 (commit)
via a79deca1f43e0c8854820ec77a1663d8bbf9a034 (commit)
via d7d291b8276c0723a3c85d322de51480c7c3a93d (commit)
via e994ac99adab1a388104d03b90544d19526a6c47 (commit)
via bf234475965f5908355435b246ac696c35d54556 (commit)
via 8ebd52397bcf105a771895c1ceb33b7ab6880887 (commit)
via 8aeac20eed5c6eb2cc9f41ada60632265cbdd556 (commit)
via c53fb9e0fab0ce043602c50e25d7fe8b94c15b79 (commit)
via 7d5d57a522489209e6b3cecfef94bab0aae4a7f5 (commit)
via f14632e1c7ddc7a8c7a7c0d1535acb003600a1d5 (commit)
via ac89a2193e0ca105b986190cb6de7112f4b1ebdd (commit)
via 980564a2c5d0b28e1b3b84da102992c45f194f1b (commit)
via 809a3785297f5460602888c2ddacec6fe0976589 (commit)
via b781e22ea6be76ece696c80de6b59ae223f3a06f (commit)
via d4939c581f629a19220bc5b469fed98462b2b7eb (commit)
via ce6f582e7e1d5b927aeee0aab3def7ab8a5cae4f (commit)
via 6fc8952ed133607f5ce317d929d731657e405edf (commit)
via f12663164dc6488cf42a8239c9a18f06244a8bd2 (commit)
via 5f93e6f5823e4ee2a25616037ace6ab6d416e581 (commit)
via 95e7cb1e1e813786b9399ef88031520717dd2dd5 (commit)
via b1b93d2d90e58fbdfe4a4f8e363a4705dbd39bd5 (commit)
via ddd7a40445fa3940f95d8e3ece0dcbb9af5910bc (commit)
via c9aeff6bf913e3a189940f7e94b6eb789318fb2c (commit)
via 5044c03a66e63b3e1fe4e0fdeec4a3f77fed0310 (commit)
via 1a0c39586b5c73f083b7dad4d3017c5c4ffe5bf4 (commit)
via 59b8a1462f59310f5207bffef324406f61ad0d63 (commit)
via cd78cc7c9d06ec84f1e272c4b5a15d377830edba (commit)
via 5de778c73972c736efa6a2f65857e4d30f1db269 (commit)
via 5d157b098bee3e8c31ad11739e50b5f6aac064af (commit)
via 765faab6cd2437a94ceec5b1685e639f72ef8627 (commit)
via 2f0303982d8c3237a10a02bc5e578fae6f6b1f66 (commit)
via 6f5021de4086494b0f693cf0d0aef28fd0a41bd5 (commit)
via e94f86dc97746ef21c641aadb56112481d1e66f5 (commit)
via dae3f71243933d142eb3c1e8d15e18e3764bfb2e (commit)
via 4482be7a145c1bd87b0793520c95478cb7d0ea54 (commit)
via 4dcf4e849f242a929ce03b5d529e0e1a63fbaeb0 (commit)
via 98378a9a3d5d5b1dc726e0a02c0057f93e79bab8 (commit)
via e265f2b3b4da3c5988374c2f3209b10eb974c66e (commit)
via 6d38a42c2b7b9b4d4ecffe75f1f3a4f0815d4ada (commit)
via 58f60163b826a04085cbc69a6be1660c37174d7c (commit)
via df439a7eb705da7aae720a43c85c462cc6235e5a (commit)
via d1baf718d0866c64252006bf61a6f0c5da353f7b (commit)
via c8803c220093c1e52a7e992fe1013927ce92c6b2 (commit)
via d9f8f46ccd5a418dcf7b5f43aeb59cd2d9d424ba (commit)
via 2858d8075d2178fa252af4e585687855e3b30dc2 (commit)
via ec38fcda26fc3a835a56dd9d50aac0650b4dc770 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (537a84cd45c915ea7b6f607b17c1cf665ed71561)
\
N -- N -- N (49fa621b6ea36479a0581df9f258ad032bc046d3)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 49fa621b6ea36479a0581df9f258ad032bc046d3
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Dec 16 12:43:35 2015 -0500
7888: Option to use multiple concurrent range requests when fetching from Azure.
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 0f98e6e..c0033d9 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -11,12 +11,14 @@ import (
"os"
"regexp"
"strings"
+ "sync"
"time"
"github.com/curoverse/azure-sdk-for-go/storage"
)
var (
+ azureMaxGetBytes int
azureStorageAccountName string
azureStorageAccountKeyFile string
azureStorageReplication int
@@ -85,6 +87,11 @@ func init() {
"azure-storage-replication",
3,
"Replication level to report to clients when data is stored in an Azure container.")
+ flag.IntVar(
+ &azureMaxGetBytes,
+ "azure-max-get-bytes",
+ BlockSize,
+ fmt.Sprintf("Maximum bytes to request in a single GET request. If smaller than %d, use multiple concurrent range requests to retrieve a block.", BlockSize))
}
// An AzureBlobVolume stores and retrieves blocks in an Azure Blob
@@ -163,20 +170,72 @@ func (v *AzureBlobVolume) Get(loc string) ([]byte, error) {
}
func (v *AzureBlobVolume) get(loc string) ([]byte, error) {
- rdr, err := v.bsClient.GetBlob(v.containerName, loc)
- if err != nil {
- return nil, v.translateError(err)
+ expectSize := BlockSize
+ if azureMaxGetBytes < BlockSize {
+ // Unfortunately the handler doesn't tell us how long the blob
+ // is expected to be, so we have to ask Azure.
+ props, err := v.bsClient.GetBlobProperties(v.containerName, loc)
+ if err != nil {
+ return nil, v.translateError(err)
+ }
+ if props.ContentLength > int64(BlockSize) || props.ContentLength < 0 {
+ return nil, fmt.Errorf("block %s invalid size %d (max %d)", loc, props.ContentLength, BlockSize)
+ }
+ expectSize = int(props.ContentLength)
}
- defer rdr.Close()
- buf := bufs.Get(BlockSize)
- n, err := io.ReadFull(rdr, buf)
- switch err {
- case nil, io.EOF, io.ErrUnexpectedEOF:
- return buf[:n], nil
- default:
- bufs.Put(buf)
- return nil, err
+
+ buf := bufs.Get(expectSize)
+ if expectSize == 0 {
+ return buf, nil
+ }
+
+ // We'll update this actualSize if/when we get the last piece.
+ actualSize := -1
+ pieces := (expectSize + azureMaxGetBytes - 1) / azureMaxGetBytes
+ errors := make([]error, pieces)
+ var wg sync.WaitGroup
+ wg.Add(pieces)
+ for p := 0; p < pieces; p++ {
+ go func(p int) {
+ defer wg.Done()
+ startPos := p * azureMaxGetBytes
+ endPos := startPos + azureMaxGetBytes
+ if endPos > expectSize {
+ endPos = expectSize
+ }
+ var rdr io.ReadCloser
+ var err error
+ if startPos == 0 && endPos == expectSize {
+ rdr, err = v.bsClient.GetBlob(v.containerName, loc)
+ } else {
+ rdr, err = v.bsClient.GetBlobRange(v.containerName, loc, fmt.Sprintf("%d-%d", startPos, endPos-1))
+ }
+ if err != nil {
+ errors[p] = err
+ return
+ }
+ defer rdr.Close()
+ n, err := io.ReadFull(rdr, buf[startPos:endPos])
+ if pieces == 1 && (err == io.ErrUnexpectedEOF || err == io.EOF) {
+ // If we don't know the actual size,
+ // and just tried reading 64 MiB, it's
+ // normal to encounter EOF.
+ } else if err != nil {
+ errors[p] = err
+ }
+ if p == pieces-1 {
+ actualSize = startPos + n
+ }
+ }(p)
}
+ wg.Wait()
+ for _, err := range errors {
+ if err != nil {
+ bufs.Put(buf)
+ return nil, v.translateError(err)
+ }
+ }
+ return buf[:actualSize], nil
}
// Compare the given data with existing stored data.
@@ -317,6 +376,7 @@ func (v *AzureBlobVolume) translateError(err error) error {
}
var keepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
+
func (v *AzureBlobVolume) isKeepBlock(s string) bool {
return keepBlockRegexp.MatchString(s)
}
diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index b8bf5cb..eacfff5 100644
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -92,6 +92,8 @@ func (h *azStubHandler) unlockAndRace() {
h.Lock()
}
+var rangeRegexp = regexp.MustCompile(`^bytes=(\d+)-(\d+)$`)
+
func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
h.Lock()
defer h.Unlock()
@@ -204,11 +206,24 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
rw.WriteHeader(http.StatusNotFound)
return
}
+ data := blob.Data
+ if rangeSpec := rangeRegexp.FindStringSubmatch(r.Header.Get("Range")); rangeSpec != nil {
+ b0, err0 := strconv.Atoi(rangeSpec[1])
+ b1, err1 := strconv.Atoi(rangeSpec[2])
+ if err0 != nil || err1 != nil || b0 >= len(data) || b1 >= len(data) || b0 > b1 {
+ rw.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", len(data)))
+ rw.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
+ return
+ }
+ rw.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", b0, b1, len(data)))
+ rw.WriteHeader(http.StatusPartialContent)
+ data = data[b0 : b1+1]
+ }
rw.Header().Set("Last-Modified", blob.Mtime.Format(time.RFC1123))
- rw.Header().Set("Content-Length", strconv.Itoa(len(blob.Data)))
+ rw.Header().Set("Content-Length", strconv.Itoa(len(data)))
if r.Method == "GET" {
- if _, err := rw.Write(blob.Data); err != nil {
- log.Printf("write %+q: %s", blob.Data, err)
+ if _, err := rw.Write(data); err != nil {
+ log.Printf("write %+q: %s", data, err)
}
}
h.unlockAndRace()
@@ -346,6 +361,26 @@ func TestAzureBlobVolumeWithGeneric(t *testing.T) {
})
}
+func TestAzureBlobVolumeConcurrentRanges(t *testing.T) {
+ defer func(b int) {
+ azureMaxGetBytes = b
+ }(azureMaxGetBytes)
+
+ defer func(t http.RoundTripper) {
+ http.DefaultTransport = t
+ }(http.DefaultTransport)
+ http.DefaultTransport = &http.Transport{
+ Dial: (&azStubDialer{}).Dial,
+ }
+ azureWriteRaceInterval = time.Millisecond
+ azureWriteRacePollTime = time.Nanosecond
+ for _, azureMaxGetBytes = range []int{2 << 22, 2<<22 - 1, 2<<22 + 1} {
+ DoGenericVolumeTests(t, func(t TB) TestableVolume {
+ return NewTestableAzureBlobVolume(t, false, azureStorageReplication)
+ })
+ }
+}
+
func TestReadonlyAzureBlobVolumeWithGeneric(t *testing.T) {
defer func(t http.RoundTripper) {
http.DefaultTransport = t
@@ -435,7 +470,7 @@ func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) {
t.Errorf("Index %+q should be empty", buf.Bytes())
}
- v.TouchWithDate(TestHash, time.Now().Add(-1982 * time.Millisecond))
+ v.TouchWithDate(TestHash, time.Now().Add(-1982*time.Millisecond))
allDone := make(chan struct{})
go func() {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list