[ARVADOS] updated: d8574bcf0fa8577f893a3ae0431b6b3845896656

Git user git at public.curoverse.com
Mon Sep 26 12:27:16 EDT 2016


Summary of changes:
 .../app/controllers/application_controller.rb      |   7 +
 apps/workbench/app/models/arvados_api_client.rb    |   1 +
 apps/workbench/app/models/container_work_unit.rb   |   4 +-
 .../app/views/projects/_show_dashboard.html.erb    |  14 +-
 apps/workbench/config/initializers/lograge.rb      |   4 +-
 .../controllers/application_controller_test.rb     |  24 +++
 .../test/controllers/projects_controller_test.rb   |  54 +++++++
 .../test/integration/application_layout_test.rb    |   4 +-
 apps/workbench/test/unit/work_unit_test.rb         |   4 +
 build/run-build-packages.sh                        |   2 +-
 .../methods/container_requests.html.textile.liquid |   1 +
 .../schema/ContainerRequest.html.textile.liquid    |   6 +
 sdk/cwl/arvados_cwl/__init__.py                    |  45 ++++--
 sdk/cwl/arvados_cwl/arv-cwl-schema.yml             |  14 ++
 sdk/cwl/arvados_cwl/arvjob.py                      |   3 +
 sdk/cwl/arvados_cwl/arvworkflow.py                 |  83 ++++++++++-
 sdk/cwl/arvados_cwl/pathmapper.py                  |  11 +-
 sdk/cwl/arvados_cwl/runner.py                      |  16 +-
 sdk/cwl/setup.py                                   |   3 +-
 sdk/cwl/tests/test_job.py                          |  68 ++++++++-
 sdk/cwl/tests/wf/scatter2.cwl                      |  56 +++++++
 sdk/cwl/tests/wf/scatter2_subwf.cwl                |  33 +++++
 sdk/go/arvados/duration.go                         |  11 +-
 sdk/go/arvadosclient/arvadosclient.go              |  20 +++
 sdk/go/arvadosclient/pool.go                       |  36 +++--
 sdk/go/arvadostest/run_servers.go                  |   9 ++
 services/api/db/structure.sql                      |   2 -
 services/api/test/fixtures/container_requests.yml  |  75 ++++++++--
 services/arv-git-httpd/auth_handler.go             |  11 +-
 services/arv-git-httpd/gitolite_test.go            |   3 +-
 services/arv-git-httpd/integration_test.go         |  14 +-
 services/arv-git-httpd/main.go                     |   8 -
 services/fuse/arvados_fuse/__init__.py             |  51 ++++---
 services/fuse/arvados_fuse/fresh.py                |  21 ++-
 services/fuse/arvados_fuse/fusedir.py              |  73 +++++-----
 services/fuse/arvados_fuse/fusefile.py             |   4 +-
 services/fuse/tests/test_cache.py                  |   4 +-
 services/fuse/tests/test_inodes.py                 |  41 +++---
 services/fuse/tests/test_mount.py                  |  47 ++++--
 services/keepproxy/keepproxy.go                    | 161 ++++++++++++---------
 .../keepproxy.service}                             |   4 +-
 services/keepproxy/usage.go                        |  82 +++++++++++
 42 files changed, 882 insertions(+), 252 deletions(-)
 create mode 100644 sdk/cwl/arvados_cwl/arv-cwl-schema.yml
 create mode 100644 sdk/cwl/tests/wf/scatter2.cwl
 create mode 100644 sdk/cwl/tests/wf/scatter2_subwf.cwl
 copy services/{arv-git-httpd/arv-git-httpd.service => keepproxy/keepproxy.service} (69%)
 create mode 100644 services/keepproxy/usage.go

       via  d8574bcf0fa8577f893a3ae0431b6b3845896656 (commit)
       via  9930af9e0670a4eaf1f0c2129406bc52b34e872b (commit)
       via  523dadebfbee9a73a21c3f78c7b4af329930d393 (commit)
       via  c2fdf5d225efe65edcab383b213eca27e2cca30e (commit)
       via  9c8cfd22f67f9becfd9e33411b61118b27eb043e (commit)
       via  1ff9f942f8e2f7830b381e7b2c4275ae71b1ae93 (commit)
       via  9bc524ae6d516f1be41cc8a7c6a76a96bbd9578c (commit)
       via  c858ed39886eceb69e54a7c7958aaf8fb4c5a27b (commit)
       via  8b7d63024652c112973d4dd82f9a5d89cc624fc7 (commit)
       via  c1429bc60d45525469ee9f73d5215814f405d8d7 (commit)
       via  678dd505394eecffe65be82a39b2303b1e83577b (commit)
       via  cc196fc783d1ee92fb166c2ca1df4134193be85e (commit)
       via  6de67b677d281b99b53ee9a25c9228523fdf7ee2 (commit)
       via  f088cbbb989c6fb2f0e87f226d1118cb015b06aa (commit)
       via  1bc10ecc9533ca8b6e3a576b4702eb5e363c6375 (commit)
       via  fe8f132725aea9816447ac8a9804b7942bc5dffd (commit)
       via  df48d4e9d64b08e0d82ffba2424fff87c2a07701 (commit)
       via  2eec8f5269cde5abc97d1dbec4cb738bcc6a92e6 (commit)
       via  916d57c9fe68a9e12472e4d174d38d93086c6529 (commit)
       via  1633e437998860e08f7e3311270b352de718dd81 (commit)
       via  0ff7b94edaaaa07932ae757c0a2b7ba3fde026cb (commit)
       via  03faf36a90f23cdf6ec2e07ffe905bc46fb5a048 (commit)
       via  8db021c5f87e21c18debc8093d83823f62131750 (commit)
       via  210889a5400ac6ef247078af14c0766f3639b988 (commit)
       via  0e7915c354852dc63c989b9e5454a11037f3b65f (commit)
       via  8b4a1d1138ace590583601621f5cebedb8868c03 (commit)
       via  f40a1843030dea9c6dc65df7a1c3b0d507b95231 (commit)
       via  c366a7dc2ef2b056d4c132f85096a10f5defb759 (commit)
       via  6dea18bd99022c09d51ef647482e080b4f316542 (commit)
       via  29efe0926dde0ee85387846f31062f4c680cb49c (commit)
       via  2e319c5e3ebd1bf61f74b01646b755fffccb984c (commit)
       via  6b88953ebd4911bc03b73bad9c354800f07ce44f (commit)
       via  cf4baf70d54b20fd232f5eecaae435df295d8f46 (commit)
       via  232cace27f60dbac0f46a9113a4b19a561689f4c (commit)
       via  bb7f064243fcd052e17815129fb1dba2f86d493d (commit)
       via  cef8879203c34fe72725afee259d42bbc0bd5a69 (commit)
       via  3a391d6693d89c0cfe5862227fcc9a0568bc775d (commit)
       via  ff71d32328a2ca555a5c6777f2ce65db50c75247 (commit)
       via  e7aa0bb4c04e9fca75fe396579f4d29ac92d9617 (commit)
       via  69963d8ac67ce1a0711cb3e92a75b2961cbc76b9 (commit)
       via  84bcaa194dd2dd767aea291cdc17e85cd802325a (commit)
       via  559729f94c6676cc6bca707e36417da412f937f5 (commit)
       via  07e339237850f463d18ce6c06446441c6684fc58 (commit)
       via  78fedce2fadbe985c5b7ec0dbe26ad8cddfb6cdb (commit)
       via  261d51d1c2551c9e7dc87a8fca23caccfc613df4 (commit)
       via  110d992e55ed6f8537abb3cb27487ae1f66ca44e (commit)
       via  b751f27e5dadf037003fe103bf170d6eef99154c (commit)
       via  98457f6fdb93116d31bec47057094041e9c47bde (commit)
       via  defaa234ca09f1c0a202a77d7660b75d3c12de02 (commit)
       via  22773dcba5c8f59a01a08c1734dd5249517808c9 (commit)
       via  64b05a6b4e2234a3a71d6978f0e1cf9d5f1b8c6e (commit)
       via  c96214468a927d56c03876de43fa16073fc26df5 (commit)
       via  4be4c3e8ee1645a59b5769ffd9ce7abdc6da03cd (commit)
       via  bc6b243539b5480493c7aa758a1272a1a42b58f1 (commit)
      from  de6dd658e9b1f27f7895337dc20195c0419799be (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit d8574bcf0fa8577f893a3ae0431b6b3845896656
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Sep 26 12:24:46 2016 -0400

    9950: Remove uses of environment variables.

diff --git a/sdk/go/arvadosclient/pool.go b/sdk/go/arvadosclient/pool.go
index 87b67c3..26b3518 100644
--- a/sdk/go/arvadosclient/pool.go
+++ b/sdk/go/arvadosclient/pool.go
@@ -10,22 +10,32 @@ import (
 // credentials. See arvados-git-httpd for an example, and sync.Pool
 // for more information about garbage collection.
 type ClientPool struct {
-	sync.Pool
-	lastErr error
+	// Initialize new clients by coping this one.
+	Prototype *ArvadosClient
+
+	pool      *sync.Pool
+	lastErr   error
+	setupOnce sync.Once
 }
 
-// MakeClientPool returns a new empty ClientPool.
+// MakeClientPool returns a new empty ClientPool, using environment
+// variables to initialize the prototype.
 func MakeClientPool() *ClientPool {
-	p := &ClientPool{}
-	p.Pool = sync.Pool{New: func() interface{} {
-		arv, err := MakeArvadosClient()
-		if err != nil {
-			p.lastErr = err
+	proto, err := MakeArvadosClient()
+	return &ClientPool{
+		Prototype: &proto,
+		lastErr:   err,
+	}
+}
+
+func (p *ClientPool) setup() {
+	p.pool = &sync.Pool{New: func() interface{} {
+		if p.lastErr != nil {
 			return nil
 		}
-		return &arv
+		c := *p.Prototype
+		return &c
 	}}
-	return p
 }
 
 // Err returns the error that was encountered last time Get returned
@@ -39,7 +49,8 @@ func (p *ClientPool) Err() error {
 // (including its ApiToken) will be just as it was when it was Put
 // back in the pool.
 func (p *ClientPool) Get() *ArvadosClient {
-	c, ok := p.Pool.Get().(*ArvadosClient)
+	p.setupOnce.Do(p.setup)
+	c, ok := p.pool.Get().(*ArvadosClient)
 	if !ok {
 		return nil
 	}
@@ -48,5 +59,6 @@ func (p *ClientPool) Get() *ArvadosClient {
 
 // Put puts an ArvadosClient back in the pool.
 func (p *ClientPool) Put(c *ArvadosClient) {
-	p.Pool.Put(c)
+	p.setupOnce.Do(p.setup)
+	p.pool.Put(c)
 }
diff --git a/sdk/go/arvadostest/run_servers.go b/sdk/go/arvadostest/run_servers.go
index 7edc482..d3b48ea 100644
--- a/sdk/go/arvadostest/run_servers.go
+++ b/sdk/go/arvadostest/run_servers.go
@@ -22,6 +22,15 @@ func ResetEnv() {
 	}
 }
 
+// APIHost returns the address:port of the current test server.
+func APIHost() string {
+	h := authSettings["ARVADOS_API_HOST"]
+	if h == "" {
+		log.Fatal("arvadostest.APIHost() was called but authSettings is not populated")
+	}
+	return h
+}
+
 // ParseAuthSettings parses auth settings from given input
 func ParseAuthSettings(authScript []byte) {
 	scanner := bufio.NewScanner(bytes.NewReader(authScript))
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index bfdb396..9f92cd1 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -20,13 +20,12 @@ type authHandler struct {
 }
 
 func (h *authHandler) setup() {
-	os.Setenv("ARVADOS_API_HOST", theConfig.Client.APIHost)
-	if theConfig.Client.Insecure {
-		os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
-	} else {
-		os.Setenv("ARVADOS_API_HOST_INSECURE", "")
+	ac, err := arvadosclient.New(&theConfig.Client)
+	if err != nil {
+		log.Fatal(err)
 	}
-	h.clientPool = arvadosclient.MakeClientPool()
+	h.clientPool = &arvadosclient.ClientPool{Prototype: ac}
+	log.Printf("%+v", h.clientPool.Prototype)
 }
 
 func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
diff --git a/services/arv-git-httpd/gitolite_test.go b/services/arv-git-httpd/gitolite_test.go
index 219eb11..74c2b8c 100644
--- a/services/arv-git-httpd/gitolite_test.go
+++ b/services/arv-git-httpd/gitolite_test.go
@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
 	check "gopkg.in/check.v1"
 )
 
@@ -44,7 +45,7 @@ func (s *GitoliteSuite) SetUpTest(c *check.C) {
 	s.tmpRepoRoot = s.gitoliteHome + "/repositories"
 	s.Config = &Config{
 		Client: arvados.Client{
-			APIHost:  os.Getenv("ARVADOS_API_HOST"),
+			APIHost:  arvadostest.APIHost(),
 			Insecure: true,
 		},
 		Listen:     ":0",
diff --git a/services/arv-git-httpd/integration_test.go b/services/arv-git-httpd/integration_test.go
index f772b4d..5e55eca 100644
--- a/services/arv-git-httpd/integration_test.go
+++ b/services/arv-git-httpd/integration_test.go
@@ -70,7 +70,7 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
 	if s.Config == nil {
 		s.Config = &Config{
 			Client: arvados.Client{
-				APIHost:  os.Getenv("ARVADOS_API_HOST"),
+				APIHost:  arvadostest.APIHost(),
 				Insecure: true,
 			},
 			Listen:     ":0",
@@ -78,13 +78,17 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
 			RepoRoot:   s.tmpRepoRoot,
 		}
 	}
+
+	// Clear ARVADOS_API_* env vars before starting up the server,
+	// to make sure arv-git-httpd doesn't use them or complain
+	// about them being missing.
+	os.Unsetenv("ARVADOS_API_HOST")
+	os.Unsetenv("ARVADOS_API_HOST_INSECURE")
+	os.Unsetenv("ARVADOS_API_TOKEN")
+
 	theConfig = s.Config
 	err = s.testServer.Start()
 	c.Assert(err, check.Equals, nil)
-
-	// Clear ARVADOS_API_TOKEN after starting up the server, to
-	// make sure arv-git-httpd doesn't use it.
-	os.Setenv("ARVADOS_API_TOKEN", "unused-token-placates-client-library")
 }
 
 func (s *IntegrationSuite) TearDownTest(c *check.C) {
diff --git a/services/arv-git-httpd/main.go b/services/arv-git-httpd/main.go
index ccee20e..40dcf32 100644
--- a/services/arv-git-httpd/main.go
+++ b/services/arv-git-httpd/main.go
@@ -63,14 +63,6 @@ func init() {
 			log.Print("Current configuration:\n", string(j))
 		}
 	}
-
-	// MakeArvadosClient returns an error if token is unset (even
-	// though we don't need to do anything requiring
-	// authentication yet). We can't do this in newArvadosClient()
-	// just before calling MakeArvadosClient(), though, because
-	// that interferes with the env var needed by "run test
-	// servers".
-	os.Setenv("ARVADOS_API_TOKEN", "xxx")
 }
 
 func main() {

commit 9930af9e0670a4eaf1f0c2129406bc52b34e872b
Merge: de6dd65 523dade
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Sep 26 09:34:45 2016 -0400

    9950: Merge branch 'master' into 9950-arv-git-httpd-config


-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list