[ARVADOS] created: 1.3.0-1552-g72eeca4fb
Git user
git at public.curoverse.com
Mon Aug 26 15:59:16 UTC 2019
at 72eeca4fb7abb97ae41e30c032e3d48074b915f7 (commit)
commit 72eeca4fb7abb97ae41e30c032e3d48074b915f7
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Mon Aug 26 11:54:43 2019 -0400
14712: Tests use cluster config
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/services/arv-git-httpd/auth_handler_test.go b/services/arv-git-httpd/auth_handler_test.go
index a8fb3acd4..568570942 100644
--- a/services/arv-git-httpd/auth_handler_test.go
+++ b/services/arv-git-httpd/auth_handler_test.go
@@ -21,7 +21,9 @@ import (
var _ = check.Suite(&AuthHandlerSuite{})
-type AuthHandlerSuite struct{}
+type AuthHandlerSuite struct {
+ cluster *arvados.Cluster
+}
func (s *AuthHandlerSuite) SetUpSuite(c *check.C) {
arvadostest.StartAPI()
@@ -38,20 +40,20 @@ func (s *AuthHandlerSuite) SetUpTest(c *check.C) {
cfg, err := config.NewLoader(nil, nil).Load()
c.Assert(err, check.Equals, nil)
- cluster, err := cfg.GetCluster("")
+ s.cluster, err = cfg.GetCluster("")
c.Assert(err, check.Equals, nil)
- cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}}
- cluster.TLS.Insecure = true
- cluster.Git.GitCommand = "/usr/bin/git"
- cluster.Git.Repositories = repoRoot
+ s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}}
+ s.cluster.TLS.Insecure = true
+ s.cluster.Git.GitCommand = "/usr/bin/git"
+ s.cluster.Git.Repositories = repoRoot
}
func (s *AuthHandlerSuite) TestPermission(c *check.C) {
h := &authHandler{handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.Printf("%v", r.URL)
io.WriteString(w, r.URL.Path)
- })}
+ }), cluster: s.cluster}
baseURL, err := url.Parse("http://git.example/")
c.Assert(err, check.IsNil)
for _, trial := range []struct {
@@ -134,7 +136,7 @@ func (s *AuthHandlerSuite) TestPermission(c *check.C) {
}
func (s *AuthHandlerSuite) TestCORS(c *check.C) {
- h := &authHandler{}
+ h := &authHandler{cluster: s.cluster}
// CORS preflight
resp := httptest.NewRecorder()
diff --git a/services/arv-git-httpd/git_handler_test.go b/services/arv-git-httpd/git_handler_test.go
index 8417b57e4..d5cb275fd 100644
--- a/services/arv-git-httpd/git_handler_test.go
+++ b/services/arv-git-httpd/git_handler_test.go
@@ -21,8 +21,7 @@ type GitHandlerSuite struct {
cluster *arvados.Cluster
}
-func (s *GitHandlerSuite) TestEnvVars(c *check.C) {
-
+func (s *GitHandlerSuite) SetUpTest(c *check.C) {
cfg, err := config.NewLoader(nil, nil).Load()
c.Assert(err, check.Equals, nil)
s.cluster, err = cfg.GetCluster("")
@@ -31,7 +30,9 @@ func (s *GitHandlerSuite) TestEnvVars(c *check.C) {
s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:80"}: arvados.ServiceInstance{}}
s.cluster.Git.GitoliteHome = "/test/ghh"
s.cluster.Git.Repositories = "/"
+}
+func (s *GitHandlerSuite) TestEnvVars(c *check.C) {
u, err := url.Parse("git.zzzzz.arvadosapi.com/test")
c.Check(err, check.Equals, nil)
resp := httptest.NewRecorder()
diff --git a/services/arv-git-httpd/integration_test.go b/services/arv-git-httpd/integration_test.go
index a6f114140..46bf8329c 100644
--- a/services/arv-git-httpd/integration_test.go
+++ b/services/arv-git-httpd/integration_test.go
@@ -61,6 +61,23 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
_, err = exec.Command("sh", "-c", "cd "+s.tmpWorkdir+" && echo work >work && git add work && git -c user.name=Foo -c user.email=Foo commit -am 'workdir: test'").CombinedOutput()
c.Assert(err, check.Equals, nil)
+ if s.cluster == nil {
+ cfg, err := config.NewLoader(nil, nil).Load()
+ c.Assert(err, check.Equals, nil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.Equals, nil)
+
+ s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}}
+ s.cluster.TLS.Insecure = true
+ s.cluster.Git.GitCommand = "/usr/bin/git"
+ s.cluster.Git.Repositories = s.tmpRepoRoot
+ s.cluster.ManagementToken = arvadostest.ManagementToken
+ }
+
+ s.testServer = &server{cluster: s.cluster}
+ err = s.testServer.Start()
+ c.Assert(err, check.Equals, nil)
+
_, err = exec.Command("git", "config",
"--file", s.tmpWorkdir+"/.git/config",
"credential.http://"+s.testServer.Addr+"/.helper",
@@ -78,27 +95,6 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
os.Unsetenv("ARVADOS_API_HOST")
os.Unsetenv("ARVADOS_API_HOST_INSECURE")
os.Unsetenv("ARVADOS_API_TOKEN")
-
- cfg, err := config.NewLoader(nil, nil).Load()
- c.Assert(err, check.Equals, nil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.Equals, nil)
-
- if s.cluster == nil {
- s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}}
- s.cluster.TLS.Insecure = true
- s.cluster.Git.GitCommand = "/usr/bin/git"
- s.cluster.Git.Repositories = s.tmpRepoRoot
- }
-
- println(s.cluster.Services.Controller.InternalURLs)
- println(arvadostest.APIHost())
- println(s.cluster.ManagementToken)
- println(arvadostest.ManagementToken)
-
- s.testServer = &server{cluster: s.cluster}
- err = s.testServer.Start()
- c.Assert(err, check.Equals, nil)
}
func (s *IntegrationSuite) TearDownTest(c *check.C) {
commit a5db96feced74279d61fe8254ed38a321342da1d
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Mon Aug 26 10:44:23 2019 -0400
14712: Fixes tests
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/services/arv-git-httpd/auth_handler_test.go b/services/arv-git-httpd/auth_handler_test.go
index 05fde03e7..a8fb3acd4 100644
--- a/services/arv-git-httpd/auth_handler_test.go
+++ b/services/arv-git-httpd/auth_handler_test.go
@@ -13,6 +13,7 @@ import (
"path/filepath"
"strings"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
@@ -34,16 +35,16 @@ func (s *AuthHandlerSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
repoRoot, err := filepath.Abs("../api/tmp/git/test")
c.Assert(err, check.IsNil)
- theConfig = &Config{
- Client: arvados.Client{
- APIHost: arvadostest.APIHost(),
- Insecure: true,
- },
- Listen: ":0",
- GitCommand: "/usr/bin/git",
- RepoRoot: repoRoot,
- ManagementToken: arvadostest.ManagementToken,
- }
+
+ cfg, err := config.NewLoader(nil, nil).Load()
+ c.Assert(err, check.Equals, nil)
+ cluster, err := cfg.GetCluster("")
+ c.Assert(err, check.Equals, nil)
+
+ cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}}
+ cluster.TLS.Insecure = true
+ cluster.Git.GitCommand = "/usr/bin/git"
+ cluster.Git.Repositories = repoRoot
}
func (s *AuthHandlerSuite) TestPermission(c *check.C) {
diff --git a/services/arv-git-httpd/git_handler_test.go b/services/arv-git-httpd/git_handler_test.go
index 0cf7de4e2..8417b57e4 100644
--- a/services/arv-git-httpd/git_handler_test.go
+++ b/services/arv-git-httpd/git_handler_test.go
@@ -10,17 +10,27 @@ import (
"net/url"
"regexp"
+ "git.curoverse.com/arvados.git/lib/config"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&GitHandlerSuite{})
-type GitHandlerSuite struct{}
+type GitHandlerSuite struct {
+ cluster *arvados.Cluster
+}
func (s *GitHandlerSuite) TestEnvVars(c *check.C) {
- theConfig = defaultConfig()
- theConfig.RepoRoot = "/"
- theConfig.GitoliteHome = "/test/ghh"
+
+ cfg, err := config.NewLoader(nil, nil).Load()
+ c.Assert(err, check.Equals, nil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.Equals, nil)
+
+ s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:80"}: arvados.ServiceInstance{}}
+ s.cluster.Git.GitoliteHome = "/test/ghh"
+ s.cluster.Git.Repositories = "/"
u, err := url.Parse("git.zzzzz.arvadosapi.com/test")
c.Check(err, check.Equals, nil)
@@ -30,7 +40,7 @@ func (s *GitHandlerSuite) TestEnvVars(c *check.C) {
URL: u,
RemoteAddr: "[::1]:12345",
}
- h := newGitHandler()
+ h := newGitHandler(s.cluster)
h.(*gitHandler).Path = "/bin/sh"
h.(*gitHandler).Args = []string{"-c", "printf 'Content-Type: text/plain\r\n\r\n'; env"}
@@ -43,7 +53,7 @@ func (s *GitHandlerSuite) TestEnvVars(c *check.C) {
c.Check(body, check.Matches, `(?ms).*^GL_BYPASS_ACCESS_CHECKS=1$.*`)
c.Check(body, check.Matches, `(?ms).*^REMOTE_HOST=::1$.*`)
c.Check(body, check.Matches, `(?ms).*^REMOTE_PORT=12345$.*`)
- c.Check(body, check.Matches, `(?ms).*^SERVER_ADDR=`+regexp.QuoteMeta(theConfig.Listen)+`$.*`)
+ c.Check(body, check.Matches, `(?ms).*^SERVER_ADDR=`+regexp.QuoteMeta("localhost:80")+`$.*`)
}
func (s *GitHandlerSuite) TestCGIErrorOnSplitHostPortError(c *check.C) {
@@ -55,7 +65,7 @@ func (s *GitHandlerSuite) TestCGIErrorOnSplitHostPortError(c *check.C) {
URL: u,
RemoteAddr: "test.bad.address.missing.port",
}
- h := newGitHandler()
+ h := newGitHandler(s.cluster)
h.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusInternalServerError)
c.Check(resp.Body.String(), check.Equals, "")
diff --git a/services/arv-git-httpd/gitolite_test.go b/services/arv-git-httpd/gitolite_test.go
index 88cd221cb..eaa7b55f8 100644
--- a/services/arv-git-httpd/gitolite_test.go
+++ b/services/arv-git-httpd/gitolite_test.go
@@ -10,8 +10,8 @@ import (
"os/exec"
"strings"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -47,16 +47,18 @@ func (s *GitoliteSuite) SetUpTest(c *check.C) {
runGitolite("gitolite", "setup", "--admin", "root")
s.tmpRepoRoot = s.gitoliteHome + "/repositories"
- s.Config = &Config{
- Client: arvados.Client{
- APIHost: arvadostest.APIHost(),
- Insecure: true,
- },
- Listen: "localhost:0",
- GitCommand: "/usr/share/gitolite3/gitolite-shell",
- GitoliteHome: s.gitoliteHome,
- RepoRoot: s.tmpRepoRoot,
- }
+
+ cfg, err := config.NewLoader(nil, nil).Load()
+ c.Assert(err, check.Equals, nil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.Equals, nil)
+
+ s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}}
+ s.cluster.TLS.Insecure = true
+ s.cluster.Git.GitCommand = "/usr/share/gitolite3/gitolite-shell"
+ s.cluster.Git.GitoliteHome = s.gitoliteHome
+ s.cluster.Git.Repositories = s.tmpRepoRoot
+
s.IntegrationSuite.SetUpTest(c)
// Install the gitolite hooks in the bare repo we made in
diff --git a/services/arv-git-httpd/integration_test.go b/services/arv-git-httpd/integration_test.go
index 53b636dc0..a6f114140 100644
--- a/services/arv-git-httpd/integration_test.go
+++ b/services/arv-git-httpd/integration_test.go
@@ -12,6 +12,7 @@ import (
"strings"
"testing"
+ "git.curoverse.com/arvados.git/lib/config"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
@@ -28,7 +29,7 @@ type IntegrationSuite struct {
tmpRepoRoot string
tmpWorkdir string
testServer *server
- Config *Config
+ cluster *arvados.Cluster
}
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
@@ -41,7 +42,7 @@ func (s *IntegrationSuite) TearDownSuite(c *check.C) {
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
- s.testServer = &server{}
+
var err error
if s.tmpRepoRoot == "" {
s.tmpRepoRoot, err = ioutil.TempDir("", "arv-git-httpd")
@@ -71,19 +72,6 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
"none").Output()
c.Assert(err, check.Equals, nil)
- if s.Config == nil {
- s.Config = &Config{
- Client: arvados.Client{
- APIHost: arvadostest.APIHost(),
- Insecure: true,
- },
- Listen: "localhost:0",
- GitCommand: "/usr/bin/git",
- RepoRoot: s.tmpRepoRoot,
- ManagementToken: arvadostest.ManagementToken,
- }
- }
-
// Clear ARVADOS_API_* env vars before starting up the server,
// to make sure arv-git-httpd doesn't use them or complain
// about them being missing.
@@ -91,7 +79,24 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
os.Unsetenv("ARVADOS_API_HOST_INSECURE")
os.Unsetenv("ARVADOS_API_TOKEN")
- theConfig = s.Config
+ cfg, err := config.NewLoader(nil, nil).Load()
+ c.Assert(err, check.Equals, nil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.Equals, nil)
+
+ if s.cluster == nil {
+ s.cluster.Services.GitHTTP.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: "localhost:0"}: arvados.ServiceInstance{}}
+ s.cluster.TLS.Insecure = true
+ s.cluster.Git.GitCommand = "/usr/bin/git"
+ s.cluster.Git.Repositories = s.tmpRepoRoot
+ }
+
+ println(s.cluster.Services.Controller.InternalURLs)
+ println(arvadostest.APIHost())
+ println(s.cluster.ManagementToken)
+ println(arvadostest.ManagementToken)
+
+ s.testServer = &server{cluster: s.cluster}
err = s.testServer.Start()
c.Assert(err, check.Equals, nil)
}
@@ -116,9 +121,7 @@ func (s *IntegrationSuite) TearDownTest(c *check.C) {
}
s.tmpWorkdir = ""
- s.Config = nil
-
- theConfig = defaultConfig()
+ s.cluster = nil
}
func (s *IntegrationSuite) RunGit(c *check.C, token, gitCmd, repo string, args ...string) error {
commit 73a68fb27a3531feed18b339ad4c66ff3d08c501
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Mon Aug 26 10:43:55 2019 -0400
14712: Cleans up arv-git arvbox setup
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/tools/arvbox/lib/arvbox/docker/cluster-config.sh b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
index 19aa6376b..951b592ea 100755
--- a/tools/arvbox/lib/arvbox/docker/cluster-config.sh
+++ b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
@@ -92,7 +92,7 @@ Clusters:
GitHTTP:
InternalURLs:
"http://localhost:${services[arv-git-httpd]}/": {}
- ExternalURL: "https://git.$localip:${services[arv-git-httpd-ssl]}/"
+ ExternalURL: "https://$localip:${services[arv-git-httpd-ssl]}/"
WebDAV:
InternalURLs:
"http://localhost:${services[keep-web]}/": {}
diff --git a/tools/arvbox/lib/arvbox/docker/service/nginx/run b/tools/arvbox/lib/arvbox/docker/service/nginx/run
index 96c0cf049..0d60e7412 100755
--- a/tools/arvbox/lib/arvbox/docker/service/nginx/run
+++ b/tools/arvbox/lib/arvbox/docker/service/nginx/run
@@ -148,7 +148,7 @@ server {
}
server {
listen *:${services[arv-git-httpd-ssl]} ssl default_server;
- server_name git.${localip};
+ server_name arvados-git-httpd;
proxy_connect_timeout 90s;
proxy_read_timeout 300s;
commit 59c608c6052d2bf47a49d6a649be5f407986f6a3
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Mon Aug 19 13:57:27 2019 -0400
14712: Updates arvbox and run_test_server with arv git cluster config
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index e79b4843a..34342059f 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -578,16 +578,11 @@ def run_arv_git_httpd():
return
stop_arv_git_httpd()
- gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
gitport = internal_port_from_config("GitHTTP")
env = os.environ.copy()
env.pop('ARVADOS_API_TOKEN', None)
logf = open(_logfilename('arv-git-httpd'), 'a')
- agh = subprocess.Popen(
- ['arv-git-httpd',
- '-repo-root='+gitdir+'/test',
- '-management-token=e687950a23c3a9bceec28c6223a06c79',
- '-address=:'+str(gitport)],
+ agh = subprocess.Popen(['arv-git-httpd'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
with open(_pidfile('arv-git-httpd'), 'w') as f:
f.write(str(agh.pid))
@@ -747,6 +742,9 @@ def setup_config():
},
"Collections": {
"TrustAllContent": True
+ },
+ "Git": {
+ "Repositories": "%s/test" % os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
}
}
}
diff --git a/tools/arvbox/lib/arvbox/docker/cluster-config.sh b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
index 34a0c2d75..19aa6376b 100755
--- a/tools/arvbox/lib/arvbox/docker/cluster-config.sh
+++ b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
@@ -90,7 +90,9 @@ Clusters:
GitSSH:
ExternalURL: "ssh://git@$localip:"
GitHTTP:
- ExternalURL: "http://$localip:${services[arv-git-httpd]}/"
+ InternalURLs:
+ "http://localhost:${services[arv-git-httpd]}/": {}
+ ExternalURL: "https://git.$localip:${services[arv-git-httpd-ssl]}/"
WebDAV:
InternalURLs:
"http://localhost:${services[keep-web]}/": {}
@@ -143,6 +145,10 @@ Clusters:
Workbench:
SecretKeyBase: $workbench_secret_key_base
ArvadosDocsite: http://$localip:${services[doc]}/
+ Git:
+ GitCommand: /usr/share/gitolite3/gitolite-shell
+ GitoliteHome: /var/lib/arvados/git
+ Repositories: /var/lib/arvados/git/repositories
EOF
/usr/local/lib/arvbox/yml_override.py /var/lib/arvados/cluster_config.yml
diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index 218727495..9d29eb9f1 100644
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -33,6 +33,7 @@ services=(
[controller-ssl]=8000
[sso]=8900
[composer]=4200
+ [arv-git-httpd-ssl]=9000
[arv-git-httpd]=9001
[keep-web]=9003
[keep-web-ssl]=9002
diff --git a/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service b/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
index 9339f2328..38522a714 100755
--- a/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/arv-git-httpd/run-service
@@ -21,8 +21,4 @@ export ARVADOS_API_HOST_INSECURE=1
export PATH="$PATH:/var/lib/arvados/git/bin"
cd ~git
-exec /usr/local/bin/arv-git-httpd \
- -address=:${services[arv-git-httpd]} \
- -git-command=/usr/share/gitolite3/gitolite-shell \
- -gitolite-home=/var/lib/arvados/git \
- -repo-root=/var/lib/arvados/git/repositories
+exec /usr/local/bin/arv-git-httpd
diff --git a/tools/arvbox/lib/arvbox/docker/service/nginx/run b/tools/arvbox/lib/arvbox/docker/service/nginx/run
index 04a1b5395..96c0cf049 100755
--- a/tools/arvbox/lib/arvbox/docker/service/nginx/run
+++ b/tools/arvbox/lib/arvbox/docker/service/nginx/run
@@ -143,6 +143,29 @@ server {
}
}
+ upstream arvados-git-httpd {
+ server localhost:${services[arv-git-httpd]};
+ }
+ server {
+ listen *:${services[arv-git-httpd-ssl]} ssl default_server;
+ server_name git.${localip};
+ proxy_connect_timeout 90s;
+ proxy_read_timeout 300s;
+
+ ssl on;
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
+ client_max_body_size 50m;
+
+ location / {
+ proxy_pass http://arvados-git-httpd;
+ proxy_set_header Host \$http_host;
+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
+
}
EOF
commit 33e1bbfb47da58c94bb705324362c4c709ac43bb
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Fri Aug 16 14:54:30 2019 -0400
14712: arv-git-httpd uses cluster config
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 3b3032afd..6c6181894 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -14,6 +14,7 @@ import (
"sync"
"time"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/auth"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
@@ -22,14 +23,21 @@ import (
type authHandler struct {
handler http.Handler
clientPool *arvadosclient.ClientPool
+ cluster *arvados.Cluster
setupOnce sync.Once
}
func (h *authHandler) setup() {
- ac, err := arvadosclient.New(&theConfig.Client)
+ client, err := arvados.NewClientFromConfig(h.cluster)
if err != nil {
log.Fatal(err)
}
+
+ ac, err := arvadosclient.New(client)
+ if err != nil {
+ log.Fatalf("Error setting up arvados client prototype %v", err)
+ }
+
h.clientPool = &arvadosclient.ClientPool{Prototype: ac}
}
@@ -161,7 +169,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
"/" + repoName + "/.git",
}
for _, dir := range tryDirs {
- if fileInfo, err := os.Stat(theConfig.RepoRoot + dir); err != nil {
+ if fileInfo, err := os.Stat(h.cluster.Git.Repositories + dir); err != nil {
if !os.IsNotExist(err) {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
@@ -173,7 +181,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
}
if rewrittenPath == "" {
log.Println("WARNING:", repoUUID,
- "git directory not found in", theConfig.RepoRoot, tryDirs)
+ "git directory not found in", h.cluster.Git.Repositories, tryDirs)
// We say "content not found" to disambiguate from the
// earlier "API says that repo does not exist" error.
statusCode, statusText = http.StatusNotFound, "content not found"
diff --git a/services/arv-git-httpd/git_handler.go b/services/arv-git-httpd/git_handler.go
index d9b08a995..0b14e81d5 100644
--- a/services/arv-git-httpd/git_handler.go
+++ b/services/arv-git-httpd/git_handler.go
@@ -10,6 +10,8 @@ import (
"net/http"
"net/http/cgi"
"os"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
)
// gitHandler is an http.Handler that invokes git-http-backend (or
@@ -20,29 +22,34 @@ type gitHandler struct {
cgi.Handler
}
-func newGitHandler() http.Handler {
+func newGitHandler(cluster *arvados.Cluster) http.Handler {
const glBypass = "GL_BYPASS_ACCESS_CHECKS"
const glHome = "GITOLITE_HTTP_HOME"
var env []string
path := os.Getenv("PATH")
- if theConfig.GitoliteHome != "" {
+ if cluster.Git.GitoliteHome != "" {
env = append(env,
- glHome+"="+theConfig.GitoliteHome,
+ glHome+"="+cluster.Git.GitoliteHome,
glBypass+"=1")
- path = path + ":" + theConfig.GitoliteHome + "/bin"
+ path = path + ":" + cluster.Git.GitoliteHome + "/bin"
} else if home, bypass := os.Getenv(glHome), os.Getenv(glBypass); home != "" || bypass != "" {
env = append(env, glHome+"="+home, glBypass+"="+bypass)
log.Printf("DEPRECATED: Passing through %s and %s environment variables. Use GitoliteHome configuration instead.", glHome, glBypass)
}
+
+ var listen arvados.URL
+ for listen = range cluster.Services.GitHTTP.InternalURLs {
+ break
+ }
env = append(env,
- "GIT_PROJECT_ROOT="+theConfig.RepoRoot,
+ "GIT_PROJECT_ROOT="+cluster.Git.Repositories,
"GIT_HTTP_EXPORT_ALL=",
- "SERVER_ADDR="+theConfig.Listen,
+ "SERVER_ADDR="+listen.Host,
"PATH="+path)
return &gitHandler{
Handler: cgi.Handler{
- Path: theConfig.GitCommand,
- Dir: theConfig.RepoRoot,
+ Path: cluster.Git.GitCommand,
+ Dir: cluster.Git.Repositories,
Env: env,
Args: []string{"http-backend"},
},
diff --git a/services/arv-git-httpd/main.go b/services/arv-git-httpd/main.go
index 74ac7ae55..3edfcf4ca 100644
--- a/services/arv-git-httpd/main.go
+++ b/services/arv-git-httpd/main.go
@@ -5,89 +5,62 @@
package main
import (
- "encoding/json"
"flag"
"fmt"
- "log"
"os"
- "regexp"
- "git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/config"
+ "git.curoverse.com/arvados.git/lib/config"
"github.com/coreos/go-systemd/daemon"
+ "github.com/ghodss/yaml"
+ log "github.com/sirupsen/logrus"
)
var version = "dev"
-// Server configuration
-type Config struct {
- Client arvados.Client
- Listen string
- GitCommand string
- RepoRoot string
- GitoliteHome string
- ManagementToken string
-}
-
-var theConfig = defaultConfig()
-
-func defaultConfig() *Config {
- return &Config{
- Listen: ":80",
- GitCommand: "/usr/bin/git",
- RepoRoot: "/var/lib/arvados/git/repositories",
- }
-}
-
func main() {
- const defaultCfgPath = "/etc/arvados/git-httpd/git-httpd.yml"
- const deprecated = " (DEPRECATED -- use config file instead)"
- flag.StringVar(&theConfig.Listen, "address", theConfig.Listen,
- "Address to listen on, \"host:port\" or \":port\"."+deprecated)
- flag.StringVar(&theConfig.GitCommand, "git-command", theConfig.GitCommand,
- "Path to git or gitolite-shell executable. Each authenticated request will execute this program with a single argument, \"http-backend\"."+deprecated)
- flag.StringVar(&theConfig.RepoRoot, "repo-root", theConfig.RepoRoot,
- "Path to git repositories."+deprecated)
- flag.StringVar(&theConfig.GitoliteHome, "gitolite-home", theConfig.GitoliteHome,
- "Value for GITOLITE_HTTP_HOME environment variable. If not empty, GL_BYPASS_ACCESS_CHECKS=1 will also be set."+deprecated)
+ logger := log.New()
+ log.SetFormatter(&log.JSONFormatter{
+ TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
+ })
- cfgPath := flag.String("config", defaultCfgPath, "Configuration file `path`.")
- dumpConfig := flag.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
- getVersion := flag.Bool("version", false, "print version information and exit.")
+ flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+ loader := config.NewLoader(os.Stdin, logger)
+ loader.SetupFlags(flags)
- flag.StringVar(&theConfig.ManagementToken, "management-token", theConfig.ManagementToken,
- "Authorization token to be included in all health check requests.")
+ dumpConfig := flags.Bool("dump-config", false, "write current configuration to stdout and exit (useful for migrating from command line flags to config file)")
+ getVersion := flags.Bool("version", false, "print version information and exit.")
- flag.Usage = usage
- flag.Parse()
+ args := loader.MungeLegacyConfigArgs(logger, os.Args[1:], "-legacy-git-httpd-config")
+ flags.Parse(args)
- // Print version information if requested
if *getVersion {
fmt.Printf("arv-git-httpd %s\n", version)
return
}
- err := config.LoadFile(theConfig, *cfgPath)
+ cfg, err := loader.Load()
if err != nil {
- h := os.Getenv("ARVADOS_API_HOST")
- if h == "" || !os.IsNotExist(err) || *cfgPath != defaultCfgPath {
- log.Fatal(err)
- }
- log.Print("DEPRECATED: No config file found, but ARVADOS_API_HOST environment variable is set. Please use a config file instead.")
- theConfig.Client.APIHost = h
- if regexp.MustCompile("^(?i:1|yes|true)$").MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) {
- theConfig.Client.Insecure = true
- }
- if j, err := json.MarshalIndent(theConfig, "", " "); err == nil {
- log.Print("Current configuration:\n", string(j))
- }
+ log.Fatal(err)
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ log.Fatal(err)
}
if *dumpConfig {
- log.Fatal(config.DumpAndExit(theConfig))
+ out, err := yaml.Marshal(cfg)
+ if err != nil {
+ log.Fatal(err)
+ }
+ _, err = os.Stdout.Write(out)
+ if err != nil {
+ log.Fatal(err)
+ }
+ return
}
- srv := &server{}
+ srv := &server{cluster: cluster}
if err := srv.Start(); err != nil {
log.Fatal(err)
}
@@ -96,7 +69,7 @@ func main() {
}
log.Printf("arv-git-httpd %s started", version)
log.Println("Listening at", srv.Addr)
- log.Println("Repository root", theConfig.RepoRoot)
+ log.Println("Repository root", cluster.Git.Repositories)
if err := srv.Wait(); err != nil {
log.Fatal(err)
}
diff --git a/services/arv-git-httpd/server.go b/services/arv-git-httpd/server.go
index 8f0d90f89..56c9765b5 100644
--- a/services/arv-git-httpd/server.go
+++ b/services/arv-git-httpd/server.go
@@ -7,22 +7,30 @@ package main
import (
"net/http"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/health"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
)
type server struct {
httpserver.Server
+ cluster *arvados.Cluster
}
func (srv *server) Start() error {
mux := http.NewServeMux()
- mux.Handle("/", &authHandler{handler: newGitHandler()})
+ mux.Handle("/", &authHandler{handler: newGitHandler(srv.cluster), cluster: srv.cluster})
mux.Handle("/_health/", &health.Handler{
- Token: theConfig.ManagementToken,
+ Token: srv.cluster.ManagementToken,
Prefix: "/_health/",
})
+
+ var listen arvados.URL
+ for listen = range srv.cluster.Services.GitHTTP.InternalURLs {
+ break
+ }
+
srv.Handler = mux
- srv.Addr = theConfig.Listen
+ srv.Addr = listen.Host
return srv.Server.Start()
}
diff --git a/services/arv-git-httpd/usage.go b/services/arv-git-httpd/usage.go
deleted file mode 100644
index 8863da686..000000000
--- a/services/arv-git-httpd/usage.go
+++ /dev/null
@@ -1,81 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-// arvados-git-httpd provides authenticated access to Arvados-hosted
-// git repositories.
-//
-// See http://doc.arvados.org/install/install-arv-git-httpd.html.
-package main
-
-import (
- "flag"
- "fmt"
- "os"
-
- "github.com/ghodss/yaml"
-)
-
-func usage() {
- c := defaultConfig()
- c.Client.APIHost = "zzzzz.arvadosapi.com:443"
- exampleConfigFile, err := yaml.Marshal(c)
- if err != nil {
- panic(err)
- }
- fmt.Fprintf(os.Stderr, `
-
-arvados-git-httpd provides authenticated access to Arvados-hosted git
-repositories.
-
-See http://doc.arvados.org/install/install-arv-git-httpd.html.
-
-Usage: arvados-git-httpd [-config path/to/arvados/git-httpd.yml]
-
-Options:
-`)
- flag.PrintDefaults()
- fmt.Fprintf(os.Stderr, `
-Example config file:
-
-%s
-
-Client.APIHost:
-
- Address (or address:port) of the Arvados API endpoint.
-
-Client.AuthToken:
-
- Unused. Normally empty, or omitted entirely.
-
-Client.Insecure:
-
- True if your Arvados API endpoint uses an unverifiable SSL/TLS
- certificate.
-
-GitCommand:
-
- Path to git or gitolite-shell executable. Each authenticated
- request will execute this program with the single argument
- "http-backend".
-
-GitoliteHome:
-
- Path to Gitolite's home directory. If a non-empty path is given,
- the CGI environment will be set up to support the use of
- gitolite-shell as a GitCommand: for example, if GitoliteHome is
- "/gh", then the CGI environment will have GITOLITE_HTTP_HOME=/gh,
- PATH=$PATH:/gh/bin, and GL_BYPASS_ACCESS_CHECKS=1.
-
-Listen:
-
- Local port to listen on. Can be "address:port" or ":port", where
- "address" is a host IP address or name and "port" is a port number
- or name.
-
-RepoRoot:
-
- Path to git repositories.
-
-`, exampleConfigFile)
-}
commit 0de3f63a136cab2227204eb16da9ea0eb9b68349
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Fri Aug 16 11:40:14 2019 -0400
14712: Adds arv-git-httpd to cluster config loading
Also removes assert path from service file
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
14712:
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 163cd87ec..3e098a002 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -419,6 +419,17 @@ Clusters:
ProviderAppID: ""
Git:
+ # Path to git or gitolite-shell executable. Each authenticated
+ # request will execute this program with the single argument http-backend"
+ GitCommand: /usr/bin/git
+
+ # Path to Gitolite's home directory. If a non-empty path is given,
+ # the CGI environment will be set up to support the use of
+ # gitolite-shell as a GitCommand: for example, if GitoliteHome is
+ # "/gh", then the CGI environment will have GITOLITE_HTTP_HOME=/gh,
+ # PATH=$PATH:/gh/bin, and GL_BYPASS_ACCESS_CHECKS=1.
+ GitoliteHome: ""
+
# Git repositories must be readable by api server, or you won't be
# able to submit crunch jobs. To pass the test suites, put a clone
# of the arvados tree in {git_repositories_dir}/arvados.git or
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index df872111d..9eb8c40c1 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -468,3 +468,44 @@ func (ldr *Loader) loadOldKeepWebConfig(cfg *arvados.Config) error {
cfg.Clusters[cluster.ClusterID] = *cluster
return nil
}
+
+const defaultGitHttpdConfigPath = "/etc/arvados/git-httpd/git-httpd.yml"
+
+type oldGitHttpdConfig struct {
+ Client *arvados.Client
+ Listen string
+ GitCommand string
+ GitoliteHome string
+ RepoRoot string
+ ManagementToken string
+}
+
+func (ldr *Loader) loadOldGitHttpdConfig(cfg *arvados.Config) error {
+ if ldr.GitHttpdPath == "" {
+ return nil
+ }
+ var oc oldGitHttpdConfig
+ err := ldr.loadOldConfigHelper("arv-git-httpd", ldr.GitHttpdPath, &oc)
+ if os.IsNotExist(err) && ldr.GitHttpdPath == defaultGitHttpdConfigPath {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ loadOldClientConfig(cluster, oc.Client)
+
+ cluster.Services.GitHTTP.InternalURLs[arvados.URL{Host: oc.Listen}] = arvados.ServiceInstance{}
+ cluster.TLS.Insecure = oc.Client.Insecure
+ cluster.ManagementToken = oc.ManagementToken
+ cluster.Git.GitCommand = oc.GitCommand
+ cluster.Git.GitoliteHome = oc.GitoliteHome
+ cluster.Git.Repositories = oc.RepoRoot
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 1eae24d84..d06ed2602 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -425,6 +425,17 @@ Clusters:
ProviderAppID: ""
Git:
+ # Path to git or gitolite-shell executable. Each authenticated
+ # request will execute this program with the single argument http-backend"
+ GitCommand: /usr/bin/git
+
+ # Path to Gitolite's home directory. If a non-empty path is given,
+ # the CGI environment will be set up to support the use of
+ # gitolite-shell as a GitCommand: for example, if GitoliteHome is
+ # "/gh", then the CGI environment will have GITOLITE_HTTP_HOME=/gh,
+ # PATH=$PATH:/gh/bin, and GL_BYPASS_ACCESS_CHECKS=1.
+ GitoliteHome: ""
+
# Git repositories must be readable by api server, or you won't be
# able to submit crunch jobs. To pass the test suites, put a clone
# of the arvados tree in {git_repositories_dir}/arvados.git or
diff --git a/lib/config/load.go b/lib/config/load.go
index 84de9b60e..7e4849393 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -35,6 +35,7 @@ type Loader struct {
CrunchDispatchSlurmPath string
WebsocketPath string
KeepproxyPath string
+ GitHttpdPath string
configdata []byte
}
@@ -66,6 +67,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
+ flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
}
@@ -143,6 +145,9 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string,
if legacyConfigArg != "-legacy-keepproxy-config" {
ldr.KeepproxyPath = ""
}
+ if legacyConfigArg != "-legacy-git-httpd-config" {
+ ldr.GitHttpdPath = ""
+ }
return munged
}
@@ -244,6 +249,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
ldr.loadOldWebsocketConfig(&cfg),
ldr.loadOldKeepproxyConfig(&cfg),
+ ldr.loadOldGitHttpdConfig(&cfg),
} {
if err != nil {
return nil, err
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 93f52f3c8..5a18972f5 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -114,6 +114,8 @@ type Cluster struct {
WebDAVCache WebDAVCacheConfig
}
Git struct {
+ GitCommand string
+ GitoliteHome string
Repositories string
}
Login struct {
diff --git a/services/arv-git-httpd/arvados-git-httpd.service b/services/arv-git-httpd/arvados-git-httpd.service
index 6f8cca856..7ac160eea 100644
--- a/services/arv-git-httpd/arvados-git-httpd.service
+++ b/services/arv-git-httpd/arvados-git-httpd.service
@@ -6,7 +6,6 @@
Description=Arvados git server
Documentation=https://doc.arvados.org/
After=network.target
-AssertPathExists=/etc/arvados/git-httpd/git-httpd.yml
# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
StartLimitInterval=0
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list