[ARVADOS] updated: 49fa621b6ea36479a0581df9f258ad032bc046d3

git at public.curoverse.com git at public.curoverse.com
Wed Dec 16 12:43:45 EST 2015


Summary of changes:
 .gitignore                                         |   7 +-
 .../controllers/container_requests_controller.rb   |   2 +
 .../app/controllers/containers_controller.rb       |   2 +
 apps/workbench/app/models/container.rb             |   3 +
 apps/workbench/app/models/container_request.rb     |   3 +
 apps/workbench/app/models/pipeline_instance.rb     |   4 +
 .../pipeline_instances/_running_component.html.erb |  29 +-
 .../_show_components_running.html.erb              |   2 +-
 .../views/pipeline_instances/_show_log.html.erb    |   3 +-
 apps/workbench/config/routes.rb                    |   2 +
 .../test/integration/anonymous_access_test.rb      |  19 +-
 apps/workbench/test/integration/jobs_test.rb       |   4 +-
 .../test/integration/pipeline_instances_test.rb    |  12 +-
 apps/workbench/test/integration_helper.rb          |   2 +-
 LICENSE-2.0.txt => sdk/cli/LICENSE-2.0.txt         |   0
 sdk/cli/arvados-cli.gemspec                        |   4 +-
 sdk/go/arvadostest/fixtures.go                     |   9 +
 sdk/go/keepclient/discover.go                      |   4 +-
 sdk/go/keepclient/keepclient.go                    |   3 +-
 sdk/go/keepclient/keepclient_test.go               |  37 +-
 sdk/go/keepclient/support.go                       |  15 +-
 LICENSE-2.0.txt => sdk/pam/LICENSE-2.0.txt         |   0
 sdk/pam/MANIFEST.in                                |   1 +
 sdk/pam/setup.py                                   |   4 +-
 LICENSE-2.0.txt => sdk/python/LICENSE-2.0.txt      |   0
 sdk/python/MANIFEST.in                             |   1 +
 sdk/python/arvados/commands/run.py                 |   2 +-
 sdk/python/arvados/keep.py                         |  12 +-
 sdk/python/setup.py                                |   3 +
 LICENSE-2.0.txt => sdk/ruby/LICENSE-2.0.txt        |   0
 sdk/ruby/arvados.gemspec                           |   3 +-
 .../arvados/v1/container_requests_controller.rb    |   6 +
 .../arvados/v1/containers_controller.rb            |   7 +
 services/api/app/models/container.rb               | 171 ++++++++++
 services/api/app/models/container_request.rb       | 175 ++++++++++
 services/api/config/routes.rb                      |   2 +
 ...0151202151426_create_containers_and_requests.rb |  59 ++++
 .../migrate/20151215134304_fix_containers_index.rb |  17 +
 services/api/db/structure.sql                      | 186 ++++++++++-
 services/api/lib/whitelist_update.rb               |  18 +
 services/api/test/fixtures/jobs.yml                |   2 +-
 services/api/test/unit/container_request_test.rb   | 371 +++++++++++++++++++++
 services/api/test/unit/container_test.rb           | 200 +++++++++++
 services/datamanager/collection/collection.go      |  24 +-
 services/datamanager/collection/collection_test.go |   6 +-
 services/datamanager/datamanager.go                |  34 +-
 services/datamanager/summary/file.go               |  49 ++-
 services/dockercleaner/MANIFEST.in                 |   1 +
 .../dockercleaner/agpl-3.0.txt                     |   0
 services/dockercleaner/setup.py                    |   3 +
 services/fuse/MANIFEST.in                          |   1 +
 agpl-3.0.txt => services/fuse/agpl-3.0.txt         |   0
 services/fuse/arvados_fuse/command.py              |  20 +-
 services/fuse/setup.py                             |   3 +
 services/fuse/tests/test_command_args.py           |  75 +++++
 services/keepstore/azure_blob_volume.go            |   5 +-
 services/keepstore/azure_blob_volume_test.go       |  26 +-
 services/keepstore/bufferpool_test.go              |   6 -
 services/keepstore/collision_test.go               |   6 -
 services/keepstore/gocheck_test.go                 |  10 +
 .../keepstore/handlers_with_generic_volume_test.go |  15 +-
 services/keepstore/keepstore_test.go               |   8 +-
 services/keepstore/pull_worker_test.go             |  11 +-
 services/keepstore/s3_volume.go                    | 312 +++++++++++++++++
 services/keepstore/s3_volume_test.go               | 133 ++++++++
 services/keepstore/volume_generic_test.go          |  76 +++--
 services/keepstore/volume_unix_test.go             |  12 +-
 agpl-3.0.txt => services/login-sync/agpl-3.0.txt   |   0
 services/login-sync/arvados-login-sync.gemspec     |   2 +-
 services/nodemanager/MANIFEST.in                   |   1 +
 agpl-3.0.txt => services/nodemanager/agpl-3.0.txt  |   0
 .../arvnodeman/computenode/driver/gce.py           |  14 +-
 services/nodemanager/arvnodeman/config.py          |  10 +-
 services/nodemanager/setup.py                      |   5 +-
 .../tests/test_computenode_driver_gce.py           |  30 ++
 services/nodemanager/tests/test_config.py          |   2 +
 tools/crunchstat-summary/.gitignore                |   2 +
 tools/crunchstat-summary/MANIFEST.in               |   1 +
 .../crunchstat-summary/agpl-3.0.txt                |   0
 tools/crunchstat-summary/bin/crunchstat-summary    |  12 +
 .../crunchstat_summary}/__init__.py                |   0
 .../crunchstat_summary/command.py                  |  14 +
 .../crunchstat_summary/summarizer.py               |  89 +++++
 .../crunchstat-summary}/gittaggers.py              |   0
 .../fuse => tools/crunchstat-summary}/setup.py     |  23 +-
 .../crunchstat-summary/tests}/__init__.py          |   0
 .../tests/logfile_20151204190335.txt.gz            | Bin 0 -> 3639 bytes
 .../tests/logfile_20151204190335.txt.gz.report     |  20 ++
 .../tests/logfile_20151210063411.txt.gz            | Bin 0 -> 3093 bytes
 .../tests/logfile_20151210063411.txt.gz.report     |  10 +
 .../tests/logfile_20151210063439.txt.gz            | Bin 0 -> 3105 bytes
 .../tests/logfile_20151210063439.txt.gz.report     |  10 +
 tools/crunchstat-summary/tests/test_examples.py    |  22 ++
 tools/keep-exercise/keep-exercise.go               |  25 ++
 94 files changed, 2276 insertions(+), 258 deletions(-)
 create mode 100644 apps/workbench/app/controllers/container_requests_controller.rb
 create mode 100644 apps/workbench/app/controllers/containers_controller.rb
 create mode 100644 apps/workbench/app/models/container.rb
 create mode 100644 apps/workbench/app/models/container_request.rb
 copy LICENSE-2.0.txt => sdk/cli/LICENSE-2.0.txt (100%)
 copy LICENSE-2.0.txt => sdk/pam/LICENSE-2.0.txt (100%)
 copy LICENSE-2.0.txt => sdk/python/LICENSE-2.0.txt (100%)
 copy LICENSE-2.0.txt => sdk/ruby/LICENSE-2.0.txt (100%)
 create mode 100644 services/api/app/controllers/arvados/v1/container_requests_controller.rb
 create mode 100644 services/api/app/controllers/arvados/v1/containers_controller.rb
 create mode 100644 services/api/app/models/container.rb
 create mode 100644 services/api/app/models/container_request.rb
 create mode 100644 services/api/db/migrate/20151202151426_create_containers_and_requests.rb
 create mode 100644 services/api/db/migrate/20151215134304_fix_containers_index.rb
 create mode 100644 services/api/lib/whitelist_update.rb
 create mode 100644 services/api/test/unit/container_request_test.rb
 create mode 100644 services/api/test/unit/container_test.rb
 create mode 100644 services/dockercleaner/MANIFEST.in
 copy agpl-3.0.txt => services/dockercleaner/agpl-3.0.txt (100%)
 copy agpl-3.0.txt => services/fuse/agpl-3.0.txt (100%)
 create mode 100644 services/keepstore/gocheck_test.go
 create mode 100644 services/keepstore/s3_volume.go
 create mode 100644 services/keepstore/s3_volume_test.go
 copy agpl-3.0.txt => services/login-sync/agpl-3.0.txt (100%)
 copy agpl-3.0.txt => services/nodemanager/agpl-3.0.txt (100%)
 create mode 100644 tools/crunchstat-summary/.gitignore
 create mode 100644 tools/crunchstat-summary/MANIFEST.in
 copy agpl-3.0.txt => tools/crunchstat-summary/agpl-3.0.txt (100%)
 create mode 100755 tools/crunchstat-summary/bin/crunchstat-summary
 copy {services/fuse/tests/performance => tools/crunchstat-summary/crunchstat_summary}/__init__.py (100%)
 create mode 100644 tools/crunchstat-summary/crunchstat_summary/command.py
 create mode 100644 tools/crunchstat-summary/crunchstat_summary/summarizer.py
 copy {services/nodemanager => tools/crunchstat-summary}/gittaggers.py (100%)
 copy {services/fuse => tools/crunchstat-summary}/setup.py (62%)
 mode change 100644 => 100755
 copy {services/fuse/tests/performance => tools/crunchstat-summary/tests}/__init__.py (100%)
 create mode 100644 tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz
 create mode 100644 tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz.report
 create mode 100644 tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz
 create mode 100644 tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz.report
 create mode 100644 tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz
 create mode 100644 tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz.report
 create mode 100644 tools/crunchstat-summary/tests/test_examples.py

  discards  537a84cd45c915ea7b6f607b17c1cf665ed71561 (commit)
       via  49fa621b6ea36479a0581df9f258ad032bc046d3 (commit)
       via  08ee1fa80cc4718bce28283c67954b70ef6d7267 (commit)
       via  d71335c6af0e312b451fdc99ecae42362ba4723d (commit)
       via  637b32482c7fd0fc14197b120b3657793646216f (commit)
       via  6248a5d1b8561dec7298e20e2858cc003df5a763 (commit)
       via  242b1cc2ca6b85f7bb9e176a786d7c484eb35c53 (commit)
       via  40158e8f7027d51ca704c3fd6039818acf2b21c0 (commit)
       via  4e5342f47855cb76c22bdb9095c7e459701833bd (commit)
       via  e74e37c42a38fbd985f35d58796ae008194e9d23 (commit)
       via  a1adf1ed6f93ce0769f307a86b6389e9e8e630a9 (commit)
       via  081e0ce2c669068f5684a5e0a30935294a90147f (commit)
       via  85782d1376ba94714088499b064ba3e43aefe722 (commit)
       via  a79deca1f43e0c8854820ec77a1663d8bbf9a034 (commit)
       via  d7d291b8276c0723a3c85d322de51480c7c3a93d (commit)
       via  e994ac99adab1a388104d03b90544d19526a6c47 (commit)
       via  bf234475965f5908355435b246ac696c35d54556 (commit)
       via  8ebd52397bcf105a771895c1ceb33b7ab6880887 (commit)
       via  8aeac20eed5c6eb2cc9f41ada60632265cbdd556 (commit)
       via  c53fb9e0fab0ce043602c50e25d7fe8b94c15b79 (commit)
       via  7d5d57a522489209e6b3cecfef94bab0aae4a7f5 (commit)
       via  f14632e1c7ddc7a8c7a7c0d1535acb003600a1d5 (commit)
       via  ac89a2193e0ca105b986190cb6de7112f4b1ebdd (commit)
       via  980564a2c5d0b28e1b3b84da102992c45f194f1b (commit)
       via  809a3785297f5460602888c2ddacec6fe0976589 (commit)
       via  b781e22ea6be76ece696c80de6b59ae223f3a06f (commit)
       via  d4939c581f629a19220bc5b469fed98462b2b7eb (commit)
       via  ce6f582e7e1d5b927aeee0aab3def7ab8a5cae4f (commit)
       via  6fc8952ed133607f5ce317d929d731657e405edf (commit)
       via  f12663164dc6488cf42a8239c9a18f06244a8bd2 (commit)
       via  5f93e6f5823e4ee2a25616037ace6ab6d416e581 (commit)
       via  95e7cb1e1e813786b9399ef88031520717dd2dd5 (commit)
       via  b1b93d2d90e58fbdfe4a4f8e363a4705dbd39bd5 (commit)
       via  ddd7a40445fa3940f95d8e3ece0dcbb9af5910bc (commit)
       via  c9aeff6bf913e3a189940f7e94b6eb789318fb2c (commit)
       via  5044c03a66e63b3e1fe4e0fdeec4a3f77fed0310 (commit)
       via  1a0c39586b5c73f083b7dad4d3017c5c4ffe5bf4 (commit)
       via  59b8a1462f59310f5207bffef324406f61ad0d63 (commit)
       via  cd78cc7c9d06ec84f1e272c4b5a15d377830edba (commit)
       via  5de778c73972c736efa6a2f65857e4d30f1db269 (commit)
       via  5d157b098bee3e8c31ad11739e50b5f6aac064af (commit)
       via  765faab6cd2437a94ceec5b1685e639f72ef8627 (commit)
       via  2f0303982d8c3237a10a02bc5e578fae6f6b1f66 (commit)
       via  6f5021de4086494b0f693cf0d0aef28fd0a41bd5 (commit)
       via  e94f86dc97746ef21c641aadb56112481d1e66f5 (commit)
       via  dae3f71243933d142eb3c1e8d15e18e3764bfb2e (commit)
       via  4482be7a145c1bd87b0793520c95478cb7d0ea54 (commit)
       via  4dcf4e849f242a929ce03b5d529e0e1a63fbaeb0 (commit)
       via  98378a9a3d5d5b1dc726e0a02c0057f93e79bab8 (commit)
       via  e265f2b3b4da3c5988374c2f3209b10eb974c66e (commit)
       via  6d38a42c2b7b9b4d4ecffe75f1f3a4f0815d4ada (commit)
       via  58f60163b826a04085cbc69a6be1660c37174d7c (commit)
       via  df439a7eb705da7aae720a43c85c462cc6235e5a (commit)
       via  d1baf718d0866c64252006bf61a6f0c5da353f7b (commit)
       via  c8803c220093c1e52a7e992fe1013927ce92c6b2 (commit)
       via  d9f8f46ccd5a418dcf7b5f43aeb59cd2d9d424ba (commit)
       via  2858d8075d2178fa252af4e585687855e3b30dc2 (commit)
       via  ec38fcda26fc3a835a56dd9d50aac0650b4dc770 (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (537a84cd45c915ea7b6f607b17c1cf665ed71561)
            \
             N -- N -- N (49fa621b6ea36479a0581df9f258ad032bc046d3)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 49fa621b6ea36479a0581df9f258ad032bc046d3
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Dec 16 12:43:35 2015 -0500

    7888: Option to use multiple concurrent range requests when fetching from Azure.

diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 0f98e6e..c0033d9 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -11,12 +11,14 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/curoverse/azure-sdk-for-go/storage"
 )
 
 var (
+	azureMaxGetBytes           int
 	azureStorageAccountName    string
 	azureStorageAccountKeyFile string
 	azureStorageReplication    int
@@ -85,6 +87,11 @@ func init() {
 		"azure-storage-replication",
 		3,
 		"Replication level to report to clients when data is stored in an Azure container.")
+	flag.IntVar(
+		&azureMaxGetBytes,
+		"azure-max-get-bytes",
+		BlockSize,
+		fmt.Sprintf("Maximum bytes to request in a single GET request. If smaller than %d, use multiple concurrent range requests to retrieve a block.", BlockSize))
 }
 
 // An AzureBlobVolume stores and retrieves blocks in an Azure Blob
@@ -163,20 +170,72 @@ func (v *AzureBlobVolume) Get(loc string) ([]byte, error) {
 }
 
 func (v *AzureBlobVolume) get(loc string) ([]byte, error) {
-	rdr, err := v.bsClient.GetBlob(v.containerName, loc)
-	if err != nil {
-		return nil, v.translateError(err)
+	expectSize := BlockSize
+	if azureMaxGetBytes < BlockSize {
+		// Unfortunately the handler doesn't tell us how long the blob
+		// is expected to be, so we have to ask Azure.
+		props, err := v.bsClient.GetBlobProperties(v.containerName, loc)
+		if err != nil {
+			return nil, v.translateError(err)
+		}
+		if props.ContentLength > int64(BlockSize) || props.ContentLength < 0 {
+			return nil, fmt.Errorf("block %s invalid size %d (max %d)", loc, props.ContentLength, BlockSize)
+		}
+		expectSize = int(props.ContentLength)
 	}
-	defer rdr.Close()
-	buf := bufs.Get(BlockSize)
-	n, err := io.ReadFull(rdr, buf)
-	switch err {
-	case nil, io.EOF, io.ErrUnexpectedEOF:
-		return buf[:n], nil
-	default:
-		bufs.Put(buf)
-		return nil, err
+
+	buf := bufs.Get(expectSize)
+	if expectSize == 0 {
+		return buf, nil
+	}
+
+	// We'll update this actualSize if/when we get the last piece.
+	actualSize := -1
+	pieces := (expectSize + azureMaxGetBytes - 1) / azureMaxGetBytes
+	errors := make([]error, pieces)
+	var wg sync.WaitGroup
+	wg.Add(pieces)
+	for p := 0; p < pieces; p++ {
+		go func(p int) {
+			defer wg.Done()
+			startPos := p * azureMaxGetBytes
+			endPos := startPos + azureMaxGetBytes
+			if endPos > expectSize {
+				endPos = expectSize
+			}
+			var rdr io.ReadCloser
+			var err error
+			if startPos == 0 && endPos == expectSize {
+				rdr, err = v.bsClient.GetBlob(v.containerName, loc)
+			} else {
+				rdr, err = v.bsClient.GetBlobRange(v.containerName, loc, fmt.Sprintf("%d-%d", startPos, endPos-1))
+			}
+			if err != nil {
+				errors[p] = err
+				return
+			}
+			defer rdr.Close()
+			n, err := io.ReadFull(rdr, buf[startPos:endPos])
+			if pieces == 1 && (err == io.ErrUnexpectedEOF || err == io.EOF) {
+				// If we don't know the actual size,
+				// and just tried reading 64 MiB, it's
+				// normal to encounter EOF.
+			} else if err != nil {
+				errors[p] = err
+			}
+			if p == pieces-1 {
+				actualSize = startPos + n
+			}
+		}(p)
 	}
+	wg.Wait()
+	for _, err := range errors {
+		if err != nil {
+			bufs.Put(buf)
+			return nil, v.translateError(err)
+		}
+	}
+	return buf[:actualSize], nil
 }
 
 // Compare the given data with existing stored data.
@@ -317,6 +376,7 @@ func (v *AzureBlobVolume) translateError(err error) error {
 }
 
 var keepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`)
+
 func (v *AzureBlobVolume) isKeepBlock(s string) bool {
 	return keepBlockRegexp.MatchString(s)
 }
diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index b8bf5cb..eacfff5 100644
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -92,6 +92,8 @@ func (h *azStubHandler) unlockAndRace() {
 	h.Lock()
 }
 
+var rangeRegexp = regexp.MustCompile(`^bytes=(\d+)-(\d+)$`)
+
 func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	h.Lock()
 	defer h.Unlock()
@@ -204,11 +206,24 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 			rw.WriteHeader(http.StatusNotFound)
 			return
 		}
+		data := blob.Data
+		if rangeSpec := rangeRegexp.FindStringSubmatch(r.Header.Get("Range")); rangeSpec != nil {
+			b0, err0 := strconv.Atoi(rangeSpec[1])
+			b1, err1 := strconv.Atoi(rangeSpec[2])
+			if err0 != nil || err1 != nil || b0 >= len(data) || b1 >= len(data) || b0 > b1 {
+				rw.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", len(data)))
+				rw.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
+				return
+			}
+			rw.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", b0, b1, len(data)))
+			rw.WriteHeader(http.StatusPartialContent)
+			data = data[b0 : b1+1]
+		}
 		rw.Header().Set("Last-Modified", blob.Mtime.Format(time.RFC1123))
-		rw.Header().Set("Content-Length", strconv.Itoa(len(blob.Data)))
+		rw.Header().Set("Content-Length", strconv.Itoa(len(data)))
 		if r.Method == "GET" {
-			if _, err := rw.Write(blob.Data); err != nil {
-				log.Printf("write %+q: %s", blob.Data, err)
+			if _, err := rw.Write(data); err != nil {
+				log.Printf("write %+q: %s", data, err)
 			}
 		}
 		h.unlockAndRace()
@@ -346,6 +361,26 @@ func TestAzureBlobVolumeWithGeneric(t *testing.T) {
 	})
 }
 
+func TestAzureBlobVolumeConcurrentRanges(t *testing.T) {
+	defer func(b int) {
+		azureMaxGetBytes = b
+	}(azureMaxGetBytes)
+
+	defer func(t http.RoundTripper) {
+		http.DefaultTransport = t
+	}(http.DefaultTransport)
+	http.DefaultTransport = &http.Transport{
+		Dial: (&azStubDialer{}).Dial,
+	}
+	azureWriteRaceInterval = time.Millisecond
+	azureWriteRacePollTime = time.Nanosecond
+	for _, azureMaxGetBytes = range []int{2 << 22, 2<<22 - 1, 2<<22 + 1} {
+		DoGenericVolumeTests(t, func(t TB) TestableVolume {
+			return NewTestableAzureBlobVolume(t, false, azureStorageReplication)
+		})
+	}
+}
+
 func TestReadonlyAzureBlobVolumeWithGeneric(t *testing.T) {
 	defer func(t http.RoundTripper) {
 		http.DefaultTransport = t
@@ -435,7 +470,7 @@ func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) {
 		t.Errorf("Index %+q should be empty", buf.Bytes())
 	}
 
-	v.TouchWithDate(TestHash, time.Now().Add(-1982 * time.Millisecond))
+	v.TouchWithDate(TestHash, time.Now().Add(-1982*time.Millisecond))
 
 	allDone := make(chan struct{})
 	go func() {

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list