[ARVADOS] updated: 1.1.4-626-gb3c1156e1
Git user
git at public.curoverse.com
Mon Jul 16 09:57:37 EDT 2018
Summary of changes:
apps/workbench/Gemfile.lock | 2 +-
.../app/controllers/work_units_controller.rb | 29 +++--
apps/workbench/app/helpers/application_helper.rb | 13 ++-
apps/workbench/app/helpers/version_helper.rb | 6 +
.../views/application/_report_issue_popup.html.erb | 6 +-
apps/workbench/config/application.default.yml | 6 +
apps/workbench/lib/app_version.rb | 15 +++
doc/_config.yml | 1 +
.../_container_scheduling_parameters.liquid | 2 +
doc/api/methods/groups.html.textile.liquid | 2 +-
.../install-dispatch.html.textile.liquid | 31 +++--
doc/install/install-composer.html.textile.liquid | 59 ++++++++++
doc/user/cwl/cwl-extensions.html.textile.liquid | 23 ++++
lib/cmd/cmd.go | 16 +--
lib/controller/handler.go | 18 +++
lib/controller/handler_test.go | 8 ++
lib/controller/proxy.go | 4 +
lib/dispatchcloud/node_size.go | 32 +++---
lib/dispatchcloud/node_size_test.go | 60 +++++-----
sdk/R/R/zzz.R | 10 ++
sdk/R/README.Rmd | 2 +
sdk/cwl/arvados_cwl/__init__.py | 4 +-
sdk/cwl/arvados_cwl/arv-cwl-schema.yml | 44 +++++++
sdk/cwl/arvados_cwl/arvcontainer.py | 17 ++-
sdk/cwl/arvados_cwl/arvjob.py | 10 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 15 ++-
sdk/cwl/arvados_cwl/context.py | 5 +
sdk/cwl/arvados_cwl/pathmapper.py | 25 +++-
sdk/cwl/arvados_cwl/runner.py | 20 +++-
sdk/cwl/arvados_cwl/util.py | 31 +++++
sdk/cwl/setup.py | 8 +-
.../collection_per_tool_packed.cwl | 126 ++++++++++++--------
sdk/cwl/tests/makes_intermediates/echo.cwl | 14 +++
.../cwl/tests/makes_intermediates/hello1.txt | 0
.../tests/makes_intermediates/run_in_single.cwl | 38 ++++++
sdk/cwl/tests/makes_intermediates/subwf.cwl | 15 +++
sdk/cwl/tests/test_container.py | 50 +++++++-
sdk/cwl/tests/test_submit.py | 36 ++++++
sdk/cwl/tests/test_util.py | 45 ++++++++
sdk/cwl/tests/wf/expect_packed.cwl | 128 ++++++++++++++-------
...ubmit_wf.cwl => submit_wf_runner_resources.cwl} | 6 +
sdk/go/arvados/byte_size.go | 91 +++++++++++++++
sdk/go/arvados/byte_size_test.go | 70 +++++++++++
sdk/go/arvados/config.go | 48 +++++++-
sdk/go/arvados/config_test.go | 37 ++++++
sdk/go/arvados/container.go | 1 +
sdk/python/arvados/__init__.py | 2 -
sdk/python/arvados/api.py | 9 +-
sdk/python/arvados/commands/keepdocker.py | 2 +-
sdk/python/arvados/keep.py | 19 ++-
sdk/python/arvados/safeapi.py | 6 +
sdk/python/setup.py | 1 +
sdk/python/tests/nginx.conf | 57 +++++++--
sdk/python/tests/run_test_server.py | 21 ++--
sdk/python/tests/test_keep_client.py | 25 ++++
services/api/Gemfile.lock | 2 +-
.../controllers/arvados/v1/schema_controller.rb | 2 +
services/api/app/models/container_request.rb | 5 +
services/api/config/application.default.yml | 5 +
services/api/lib/app_version.rb | 15 +++
services/api/lib/crunch_dispatch.rb | 8 +-
.../arvados/v1/schema_controller_test.rb | 14 ++-
services/api/test/unit/container_request_test.rb | 5 +
.../crunch-dispatch-slurm_test.go | 16 +--
services/crunch-run/crunchrun.go | 123 +++++++++++++++++---
services/crunch-run/crunchrun_test.go | 68 ++++++++++-
services/crunch-run/logging_test.go | 2 +-
services/keep-web/cache.go | 2 +-
services/keep-web/cadaver_test.go | 2 +-
services/keep-web/handler.go | 4 +
services/keep-web/handler_test.go | 2 +-
.../arvnodeman/computenode/dispatch/slurm.py | 2 +-
services/nodemanager/arvnodeman/jobqueue.py | 2 +-
services/nodemanager/arvnodeman/nodelist.py | 2 +-
services/nodemanager/setup.py | 4 +-
services/nodemanager/tests/integration_test.py | 2 +-
.../tests/test_computenode_dispatch_slurm.py | 14 +--
services/nodemanager/tests/test_jobqueue.py | 12 +-
services/nodemanager/tests/test_nodelist.py | 4 +-
vendor/vendor.json | 60 ++++++++++
80 files changed, 1479 insertions(+), 269 deletions(-)
create mode 100644 doc/install/install-composer.html.textile.liquid
create mode 100644 sdk/R/R/zzz.R
create mode 100644 sdk/cwl/arvados_cwl/util.py
create mode 100644 sdk/cwl/tests/makes_intermediates/echo.cwl
copy apps/workbench/app/mailers/.gitkeep => sdk/cwl/tests/makes_intermediates/hello1.txt (100%)
create mode 100644 sdk/cwl/tests/makes_intermediates/run_in_single.cwl
create mode 100644 sdk/cwl/tests/makes_intermediates/subwf.cwl
create mode 100644 sdk/cwl/tests/test_util.py
copy sdk/cwl/tests/wf/{submit_wf.cwl => submit_wf_runner_resources.cwl} (81%)
create mode 100644 sdk/go/arvados/byte_size.go
create mode 100644 sdk/go/arvados/byte_size_test.go
create mode 100644 sdk/go/arvados/config_test.go
via b3c1156e1d4ed9f15a36bc51f2cd125a65aab3d3 (commit)
via a93ef946eb1e73ee190ea4ff19c4f9278235530c (commit)
via db5107dca09b786374f06a35abb51ffc3f032abd (commit)
via 50cbdcbd67f8b0da06d3b188e7bfdea3963661a7 (commit)
via f9a05f61abdf33891b09d62205d009d1cae73d1b (commit)
via 224ac505b1162837f2f84fe2735a959d71bd5ce9 (commit)
via f0d62ea064d32b980f723e37972788cbd693c2ff (commit)
via e7870334a9f2c44fcd8580d3e75074df216c647a (commit)
via f62cff30c9948815d3cf2b00294da01c146cb8f2 (commit)
via 751cd3892cbcf7ecb65ec416dba52ee84a2fee2a (commit)
via d640a546f2cd55cfbd0a959bcfb84be1eecdd6a6 (commit)
via 3712b5943d9d32346552cb91ec5c4690e14485b9 (commit)
via 2ac43c8c20d917376b2b92c09c46fa5641d054f8 (commit)
via 02d7f4e820b65d83a5e3709dc14b6d72b9f5ab49 (commit)
via 8e693a9981f03d229ff2bff7dd6e5d06e0790c19 (commit)
via ac6a1a232a670d10ce12d19e6d2a7900d353943a (commit)
via e0784f339e794a64108f7c18e478d44e5751cc19 (commit)
via 4369714821950366db98a54e4b62fdb5d09951a6 (commit)
via be0cdc7814a49fa093b86b698a9756971ba80fcf (commit)
via 8ae6680ebae91af9b0aafd6c9cfe5e5fb97f8b12 (commit)
via 5b0cb2c6292e8784b7a54b75b444e47a09c10a05 (commit)
via 688319dca9cbbb27452a0062943bb38b6935254e (commit)
via eead78e2eb11528af8cb862dcb6f9a41737a8a14 (commit)
via cda50fb57f9b18cb769432876433b52f484692bd (commit)
via 64937458c5bc2918fa679c97478b2931bd26cfc1 (commit)
via 08540a58a67f2d575b1e15338c6d49b8b74e2c38 (commit)
via 2c87b580a87a55010da626d352307343f75d6d3a (commit)
via e2f50c08c4fc6da7065be222c2620264b521897f (commit)
via 6b495b26e49bb32eccaddffea36d91f34d3ba6f8 (commit)
via b478b8a0b4263b0e54060e3fecaab9b6ecdcb085 (commit)
via 33c10053b22fd5065516eb7df4c58b55a70d490c (commit)
via c405f9adab08daa3d4edef9cddd5453a79446c7b (commit)
via 6dd60d7c70a63ac884515387cd8c92bb97433e1b (commit)
via 710399e0355ff8ed016a9d3c830730af2e228ad5 (commit)
via 81fa59a1e7797da8d420a94216bd28031178ae51 (commit)
via f339946832e0bb7ad175acaf59733445e6915f7a (commit)
via e5a3fb0a69df6c27dc567949a2e64d0e7da65384 (commit)
via 04c664d6150683e3258168eecc4440a635ef8b16 (commit)
via 951c8a79bca7224fdf8c50463d0c9a60b43ce930 (commit)
via 9b16a843a641159867ea390c4c2384b320183b15 (commit)
via 77cfd1ee241b4a2d408bd3ec5ebc69f34c7496f5 (commit)
via d3e2fb85a39562e38ea67f84e9f065815450294a (commit)
via b2a50a8539be09b73c5d65719e8eae4b2a15273b (commit)
via bd55c808de1d4d25fea7cc047956c45ec9752fa5 (commit)
via a8d2a183ac7246bf368d40dba1af226a4968a9c9 (commit)
via 69138b061aca4e0cbbb7dad080158205f6734e77 (commit)
via 535856c28a12bb07dc986b980b0f4ccfdfd25640 (commit)
via 335ee76030e85fa6ac4da79b598cf4c4a212443d (commit)
via cd6d6f1f15bc1452a09c16b1a3524b5b289c6100 (commit)
via c584cb69cdcfb377deed94745785330562a54ae3 (commit)
via 3f278cea46034660149403d68e05f0f450330854 (commit)
via 3a7481703542b0c95d5a23cb45b0d2e7c3ac4c9e (commit)
via 66c644efeabea073428288378a39a8e988e5b69b (commit)
via 9b6abcd0448567146b471ad02162d33fd4b1d5a8 (commit)
via 5c4d9d38dcee73a7ffb6221c80f707c3924da64f (commit)
via ee4c01999aff1ebc1e2dc338a70d1d7b812c633c (commit)
via 380e4da5aab5d24d0e90ea27880974c232538fbf (commit)
via a12e4d36901ab3b3027f1168f1ea06b26122829e (commit)
via c1ab715afeb9c095efed7e6064ce10021c067c35 (commit)
via 0695b41ae52b87adcdf49c6411f0da353d7213a9 (commit)
via 816764a283c2cbf2d41b4582113065922b99bd52 (commit)
via 8f0815985dfb74fc1c7468426359ac3fbbd8c12d (commit)
via 1f9519fba9a34f2a596c683ed6395b2e291935b7 (commit)
via 4a78e8e91fdad38e567fef0cd43aa8cb6bd33580 (commit)
via c425f01b6dd63dfe5d99a19f57c8155c2f0fb195 (commit)
via 3813e892be7f3136e5b14580e7ee64d6c786bf8e (commit)
via 4a2dc82a1acce855151928abe0030e1dd7dbf728 (commit)
via 9a9930cebb7f36131490d0f144e23f5066e7f01d (commit)
via d128fe6bc8da01fe8e7829db988819dd1c159298 (commit)
via 76a2c8a1eb1771b5255e8582b7e9823d072e8ef4 (commit)
via 62f6a306eb965cee6bd814b194bf5ce3bb910116 (commit)
via 11ab2e23511c7e8962e0110c3aad44b74fea2dbd (commit)
via 7d4123813b42d8f6478e239895e62f6f600a1c6f (commit)
via 596884ea30c551d7c2699b2cf3fdea2cede3101a (commit)
via bd059862d1dd7e68642dd365f0a536621ff6735f (commit)
via 0765fb8b19dd7b76ec5d2d05edda3f4fd4347194 (commit)
via 286f7fb2dbdcd860275a24f3480e938b5bd414ec (commit)
via 57fd9fa6bf0ee3062d7d38aceb7e97543791d241 (commit)
via 9df6d2c2152b5b1968649c970664c4f69d9e92e8 (commit)
via 905a7305413f57e3188a66fe55d35cecc484b1f2 (commit)
via c7414891876cd6d0d02abe71c7376b8104bfbc32 (commit)
via 02763280f54fd0c2a499285f8ce6afcbd8b9e082 (commit)
via 988c59f51aa579ce8bf0eab1cc729e05a5ee5631 (commit)
via 088268fc00cb466c4b4576a2dbbaff720cb3e3af (commit)
via cdd57a73f2f90d6b14b8652ab94adc14e4b99c8b (commit)
via 92ac046a8a7a0f5a3a88b8656bb171701af840a0 (commit)
via 5d340bee3506b37f0d5f0b695dfa37661ad0fbb6 (commit)
from 5eda2f0ee6fcab67d3a74517a85946435931c67b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b3c1156e1d4ed9f15a36bc51f2cd125a65aab3d3
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Jul 16 09:55:51 2018 -0400
13493: Fix proxy-redirect implementation.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 69b186616..25799aae9 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -58,6 +58,8 @@ func (h *Handler) CheckHealth() error {
return err
}
+func neverRedirect(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
+
func (h *Handler) setup() {
mux := http.NewServeMux()
mux.Handle("/_health/", &health.Handler{
@@ -72,21 +74,18 @@ func (h *Handler) setup() {
sc := *arvados.DefaultSecureClient
sc.Timeout = time.Duration(h.Cluster.HTTPRequestTimeout)
+ sc.CheckRedirect = neverRedirect
h.secureClient = &sc
ic := *arvados.InsecureHTTPClient
ic.Timeout = time.Duration(h.Cluster.HTTPRequestTimeout)
+ ic.CheckRedirect = neverRedirect
h.insecureClient = &ic
h.proxy = &proxy{
Name: "arvados-controller",
RequestTimeout: time.Duration(h.Cluster.HTTPRequestTimeout),
}
-
- // Changing the global isn't the right way to do this, but a
- // proper solution would conflict with an impending 13493
- // merge anyway, so this will do for now.
- arvados.InsecureHTTPClient.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
}
var errDBConnection = errors.New("database connection error")
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index 2f9280e11..963fd1159 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -124,9 +124,9 @@ func (s *HandlerSuite) TestProxyNotFound(c *check.C) {
}
func (s *HandlerSuite) TestProxyRedirect(c *check.C) {
- req := httptest.NewRequest("GET", "https://example.org:1234/login?return_to=foo", nil)
+ req := httptest.NewRequest("GET", "https://0.0.0.0:1/login?return_to=foo", nil)
resp := httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusFound)
- c.Check(resp.Header().Get("Location"), check.Matches, `https://example\.org:1234/auth/joshid\?return_to=foo&?`)
+ c.Check(resp.Header().Get("Location"), check.Matches, `https://0.0.0.0:1/auth/joshid\?return_to=foo&?`)
}
diff --git a/lib/controller/proxy.go b/lib/controller/proxy.go
index 1ff383d89..712071bef 100644
--- a/lib/controller/proxy.go
+++ b/lib/controller/proxy.go
@@ -46,6 +46,9 @@ func (p *proxy) Do(w http.ResponseWriter, reqIn *http.Request, urlOut *url.URL,
xff = xffIn + "," + xff
}
hdrOut.Set("X-Forwarded-For", xff)
+ if hdrOut.Get("X-Forwarded-Proto") == "" {
+ hdrOut.Set("X-Forwarded-Proto", reqIn.URL.Scheme)
+ }
hdrOut.Add("Via", reqIn.Proto+" arvados-controller")
ctx := reqIn.Context()
@@ -58,6 +61,7 @@ func (p *proxy) Do(w http.ResponseWriter, reqIn *http.Request, urlOut *url.URL,
reqOut := (&http.Request{
Method: reqIn.Method,
URL: urlOut,
+ Host: reqIn.Host,
Header: hdrOut,
Body: reqIn.Body,
}).WithContext(ctx)
commit a93ef946eb1e73ee190ea4ff19c4f9278235530c
Merge: 5eda2f0ee db5107dca
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Jul 16 09:46:23 2018 -0400
13493: Merge branch 'master' into 13493-federation-proxy
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --cc lib/controller/handler.go
index c50f98273,a1a69a88e..69b186616
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@@ -50,61 -59,75 +64,66 @@@ func (h *Handler) setup()
Token: h.Cluster.ManagementToken,
Prefix: "/_health/",
})
- mux.Handle("/", http.HandlerFunc(h.proxyRailsAPI))
+ hs := http.NotFoundHandler()
+ hs = prepend(hs, h.proxyRailsAPI)
+ hs = prepend(hs, h.proxyRemoteCluster)
+ mux.Handle("/", hs)
h.handlerStack = mux
+ sc := *arvados.DefaultSecureClient
+ sc.Timeout = time.Duration(h.Cluster.HTTPRequestTimeout)
+ h.secureClient = &sc
+
+ ic := *arvados.InsecureHTTPClient
+ ic.Timeout = time.Duration(h.Cluster.HTTPRequestTimeout)
+ h.insecureClient = &ic
+
+ h.proxy = &proxy{
+ Name: "arvados-controller",
+ RequestTimeout: time.Duration(h.Cluster.HTTPRequestTimeout),
+ }
++
+ // Changing the global isn't the right way to do this, but a
+ // proper solution would conflict with an impending 13493
+ // merge anyway, so this will do for now.
+ arvados.InsecureHTTPClient.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
}
-// headers that shouldn't be forwarded when proxying. See
-// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers
-var dropHeaders = map[string]bool{
- "Connection": true,
- "Keep-Alive": true,
- "Proxy-Authenticate": true,
- "Proxy-Authorization": true,
- "TE": true,
- "Trailer": true,
- "Transfer-Encoding": true,
- "Upgrade": true,
-}
+var errDBConnection = errors.New("database connection error")
-func (h *Handler) proxyRailsAPI(w http.ResponseWriter, reqIn *http.Request) {
- urlOut, err := findRailsAPI(h.Cluster, h.NodeProfile)
- if err != nil {
- httpserver.Error(w, err.Error(), http.StatusInternalServerError)
- return
- }
- urlOut = &url.URL{
- Scheme: urlOut.Scheme,
- Host: urlOut.Host,
- Path: reqIn.URL.Path,
- RawPath: reqIn.URL.RawPath,
- RawQuery: reqIn.URL.RawQuery,
+func (h *Handler) db(req *http.Request) (*sql.DB, error) {
+ h.pgdbMtx.Lock()
+ defer h.pgdbMtx.Unlock()
+ if h.pgdb != nil {
+ return h.pgdb, nil
}
- // Copy headers from incoming request, then add/replace proxy
- // headers like Via and X-Forwarded-For.
- hdrOut := http.Header{}
- for k, v := range reqIn.Header {
- if !dropHeaders[k] {
- hdrOut[k] = v
- }
+ db, err := sql.Open("postgres", h.Cluster.PostgreSQL.Connection.String())
+ if err != nil {
+ httpserver.Logger(req).WithError(err).Error("postgresql connect failed")
+ return nil, errDBConnection
}
- xff := reqIn.RemoteAddr
- if xffIn := reqIn.Header.Get("X-Forwarded-For"); xffIn != "" {
- xff = xffIn + "," + xff
+ if p := h.Cluster.PostgreSQL.ConnectionPool; p > 0 {
+ db.SetMaxOpenConns(p)
}
- hdrOut.Set("X-Forwarded-For", xff)
- if hdrOut.Get("X-Forwarded-Proto") == "" {
- hdrOut.Set("X-Forwarded-Proto", reqIn.URL.Scheme)
+ if err := db.Ping(); err != nil {
+ httpserver.Logger(req).WithError(err).Error("postgresql connect succeeded but ping failed")
+ return nil, errDBConnection
}
- hdrOut.Add("Via", reqIn.Proto+" arvados-controller")
+ h.pgdb = db
+ return db, nil
+}
- ctx := reqIn.Context()
- if timeout := h.Cluster.HTTPRequestTimeout; timeout > 0 {
- var cancel context.CancelFunc
- ctx, cancel = context.WithDeadline(ctx, time.Now().Add(time.Duration(timeout)))
- defer cancel()
- }
+type middlewareFunc func(http.ResponseWriter, *http.Request, http.Handler)
+
+func prepend(next http.Handler, middleware middlewareFunc) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ middleware(w, req, next)
+ })
+}
- reqOut := (&http.Request{
- Method: reqIn.Method,
- URL: urlOut,
- Host: reqIn.Host,
- Header: hdrOut,
- Body: reqIn.Body,
- }).WithContext(ctx)
- resp, err := arvados.InsecureHTTPClient.Do(reqOut)
+func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next http.Handler) {
+ urlOut, insecure, err := findRailsAPI(h.Cluster, h.NodeProfile)
if err != nil {
httpserver.Error(w, err.Error(), http.StatusInternalServerError)
return
diff --cc sdk/go/arvados/config.go
index 608bc223b,353901855..6edd18418
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@@ -52,29 -54,8 +54,29 @@@ type Cluster struct
ClusterID string `json:"-"`
ManagementToken string
NodeProfiles map[string]NodeProfile
- InstanceTypes []InstanceType
+ InstanceTypes InstanceTypeMap
HTTPRequestTimeout Duration
+ RemoteClusters map[string]RemoteCluster
+ PostgreSQL PostgreSQL
+}
+
+type PostgreSQL struct {
+ Connection PostgreSQLConnection
+ ConnectionPool int
+}
+
+type PostgreSQLConnection map[string]string
+
+type RemoteCluster struct {
+ // API endpoint host or host:port; default is {id}.arvadosapi.com
+ Host string
+ // Perform a proxy request when a local client requests an
+ // object belonging to this remote.
+ Proxy bool
+ // Scheme, default "https". Can be set to "http" for testing.
+ Scheme string
+ // Disable TLS verify. Can be set to true for testing.
+ Insecure bool
}
type InstanceType struct {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list