[ARVADOS] updated: 3ceb42c882b32fb9e4ef79679576254f98cdfb3d
Git user
git at public.curoverse.com
Thu Apr 6 16:15:17 EDT 2017
Summary of changes:
.../methods/container_requests.html.textile.liquid | 2 +
sdk/go/keepclient/support.go | 10 ++
services/api/app/models/arvados_model.rb | 53 +++++-----
services/api/app/models/collection.rb | 40 ++++---
services/api/app/models/container_request.rb | 85 ++++++++-------
...0012505_add_output_ttl_to_container_requests.rb | 5 +
services/api/db/structure.sql | 7 +-
services/api/test/unit/collection_test.rb | 23 +++--
services/api/test/unit/container_request_test.rb | 115 ++++++++++++++++++---
services/crunch-run/crunchrun.go | 32 +++---
services/crunch-run/crunchrun_test.go | 21 ++--
.../arvnodeman/computenode/dispatch/__init__.py | 10 +-
.../arvnodeman/computenode/driver/__init__.py | 17 +++
.../arvnodeman/computenode/driver/gce.py | 12 +--
services/nodemanager/arvnodeman/daemon.py | 2 +-
.../tests/test_computenode_driver_gce.py | 13 +--
16 files changed, 286 insertions(+), 161 deletions(-)
create mode 100644 services/api/db/migrate/20170330012505_add_output_ttl_to_container_requests.rb
via 3ceb42c882b32fb9e4ef79679576254f98cdfb3d (commit)
via 8fd18fc4a32797fc3c6255099fc253d7aede12b5 (commit)
via ae69864e07d416d10879e00da416918ac021893f (commit)
via b50a3bcb38cf9e79416d8cc9b0d8b66249b3d473 (commit)
via 4bb024ae6903552c29613c851312f934dc6174b9 (commit)
via a03ce4056e503710caa1e95d315b92fb74c96abf (commit)
via 77f5a84ccc2b14438286ed05c6af183b8d8be605 (commit)
via 4a35e06b098bdd44a24fcaf77921aea5f371c84b (commit)
via 07c92074fe2b493dc8cd0abdad154e5b36d4adfc (commit)
via 1654f776b55bbb861f72c0dd3118ad1beec5db31 (commit)
via ff3bb22d4b2bff5666907a6eeb6cd68cd3cbe22b (commit)
via b651984ff68c6f8add9b99427b205a2cb7c87462 (commit)
via 65121f8db54a1ed15207d050e1f48c5fc26d646b (commit)
via 4437774e863465c0daa41dfd9716174e18d93122 (commit)
via a12ed889f9d0106ea26d0e2d6ff1f74e9ab14aac (commit)
via c507b0b072ad62c0087d059aedeaae8bae9b715f (commit)
from c3ae1e2f54d4199a9521bf3d4d515bcbb0711989 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3ceb42c882b32fb9e4ef79679576254f98cdfb3d
Author: radhika <radhika at curoverse.com>
Date: Thu Apr 6 16:13:38 2017 -0400
8465: do not wait in the go routine while doing io.Copy of stdin and also invoke runner.stop on error.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index f22680f..c8f77f4 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -50,7 +50,6 @@ var ErrCancelled = errors.New("Cancelled")
type IKeepClient interface {
PutHB(hash string, buf []byte) (string, int, error)
ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error)
- CollectionFileReader(collection map[string]interface{}, filename string) (keepclient.Reader, error)
}
// NewLogWriter is a factory function to create a new log writer.
@@ -687,7 +686,7 @@ func (runner *ContainerRunner) AttachStreams() (err error) {
return fmt.Errorf("While getting stding collection: %v", err)
}
- stdinRdr, err = runner.Kc.CollectionFileReader(map[string]interface{}{"manifest_text": stdinColl.ManifestText}, stdinMnt.Path)
+ stdinRdr, err = runner.Kc.ManifestFileReader(manifest.Manifest{Text: stdinColl.ManifestText}, stdinMnt.Path)
if os.IsNotExist(err) {
return fmt.Errorf("stdin collection path not found: %v", stdinMnt.Path)
} else if err != nil {
@@ -740,30 +739,23 @@ func (runner *ContainerRunner) AttachStreams() (err error) {
runner.Stderr = NewThrottledLogger(runner.NewLogWriter("stderr"))
if stdinRdr != nil {
- copyErrC := make(chan error)
go func() {
_, err := io.Copy(response.Conn, stdinRdr)
- copyErrC <- err
- close(copyErrC)
+ if err != nil {
+ runner.CrunchLog.Print("While writing stdin collection to docker container %q", err)
+ runner.stop()
+ }
+ response.Conn.Close()
}()
-
- copyErr := <-copyErrC
- if copyErr != nil {
- return fmt.Errorf("While writing stdin collection to docker container %q", copyErr)
- }
} else if len(stdinJson) != 0 {
- copyErrC := make(chan error)
go func() {
- jsonRdr := bytes.NewReader(stdinJson)
- _, err := io.Copy(response.Conn, jsonRdr)
- copyErrC <- err
- close(copyErrC)
+ _, err := io.Copy(response.Conn, bytes.NewReader(stdinJson))
+ if err != nil {
+ runner.CrunchLog.Print("While writing stdin json to docker container %q", err)
+ runner.stop()
+ }
+ response.Conn.Close()
}()
-
- copyErr := <-copyErrC
- if copyErr != nil {
- return fmt.Errorf("While writing stdin json to docker container %q", copyErr)
- }
}
go runner.ProcessDockerAttach(response.Reader)
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index f124359..3062fc0 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -103,6 +103,10 @@ func (m *MockConn) Write(b []byte) (int, error) {
return len(b), nil
}
+func (m *MockConn) Close() error {
+ return nil
+}
+
func NewMockConn() *MockConn {
c := &MockConn{}
return c
@@ -300,12 +304,7 @@ func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename s
rdr := ioutil.NopCloser(&bytes.Buffer{})
client.Called = true
return FileWrapper{rdr, 1321984}, nil
- }
- return nil, nil
-}
-
-func (client *KeepTestClient) CollectionFileReader(collection map[string]interface{}, filename string) (keepclient.Reader, error) {
- if filename == "/file1_in_main.txt" {
+ } else if filename == "/file1_in_main.txt" {
rdr := ioutil.NopCloser(strings.NewReader("foo"))
client.Called = true
return FileWrapper{rdr, 3}, nil
@@ -392,10 +391,6 @@ func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename stri
return nil, errors.New("KeepError")
}
-func (KeepErrorTestClient) CollectionFileReader(c map[string]interface{}, f string) (keepclient.Reader, error) {
- return nil, errors.New("KeepError")
-}
-
type KeepReadErrorTestClient struct{}
func (KeepReadErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
@@ -424,10 +419,6 @@ func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename
return ErrorReader{}, nil
}
-func (KeepReadErrorTestClient) CollectionFileReader(c map[string]interface{}, f string) (keepclient.Reader, error) {
- return ErrorReader{}, nil
-}
-
func (s *TestSuite) TestLoadImageArvError(c *C) {
// (1) Arvados error
cr := NewContainerRunner(ArvErrorTestClient{}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
commit 8fd18fc4a32797fc3c6255099fc253d7aede12b5
Merge: ae69864 b50a3bc
Author: radhika <radhika at curoverse.com>
Date: Thu Apr 6 14:28:57 2017 -0400
Merge branch 'master' into 8465-stdin-redirection
commit ae69864e07d416d10879e00da416918ac021893f
Author: radhika <radhika at curoverse.com>
Date: Thu Apr 6 14:28:00 2017 -0400
8465: test clean up
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 8852bff..f124359 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -306,7 +306,7 @@ func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename s
func (client *KeepTestClient) CollectionFileReader(collection map[string]interface{}, filename string) (keepclient.Reader, error) {
if filename == "/file1_in_main.txt" {
- rdr := ioutil.NopCloser(strings.NewReader("foo")) // ioutil.NopCloser(&bytes.Buffer{})
+ rdr := ioutil.NopCloser(strings.NewReader("foo"))
client.Called = true
return FileWrapper{rdr, 3}, nil
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list