[ARVADOS] updated: 1.1.1-250-g4ce855e
Git user
git at public.curoverse.com
Thu Dec 14 11:07:49 EST 2017
Summary of changes:
build/run-build-docker-jobs-image.sh | 19 +-
doc/api/methods.html.textile.liquid | 27 ++-
sdk/go/arvadostest/fixtures.go | 2 +-
.../api/app/controllers/application_controller.rb | 2 +-
...53352_add_gin_index_to_collection_properties.rb | 8 +
services/api/db/structure.sql | 10 +
services/api/lib/record_filters.rb | 201 ++++++++++++++-------
services/api/test/fixtures/collections.yml | 98 ++++++++++
.../api/test/functional/arvados/v1/filters_test.rb | 128 +++++++++++++
services/api/test/integration/cross_origin_test.rb | 2 +-
services/arv-git-httpd/auth_handler.go | 60 +++---
services/arv-git-httpd/auth_handler_test.go | 116 ++++++++++++
.../arvnodeman/computenode/dispatch/transitions.py | 15 +-
services/nodemanager/arvnodeman/daemon.py | 5 +-
services/nodemanager/arvnodeman/nodelist.py | 4 +-
.../nodemanager/tests/test_computenode_dispatch.py | 7 +
16 files changed, 589 insertions(+), 115 deletions(-)
create mode 100644 services/api/db/migrate/20171212153352_add_gin_index_to_collection_properties.rb
discards 08edbfda915126a1e142fd72d81be17318e59889 (commit)
discards f96fbd63314748dbe389ccb1b5b37f3fae4688ac (commit)
discards d98fbd3931822993fd80ef605ed9d14d777274b1 (commit)
via 4ce855e6b1368cc6ce95c27da2238fa39060c069 (commit)
via 4343e1f1035baaa22f117d119c6e6bc35b1e385b (commit)
via a6c6892623dcd1ca882478a06f274a19d7951cb8 (commit)
via 1d2020d448c45a1bb8a7b1f6aa04ed1d87dc353b (commit)
via 314f9396cd140456a5e5a6d29e4d7741de0181f4 (commit)
via 9dd3035e4c844d2f94aac8e1e2a37f20b42594ea (commit)
via 6d4667c55fb00fba62df76e64c10a87ebb522eaf (commit)
via 335b2d29f9953f6e7767223cb71d1d0f313c9043 (commit)
via e768a05df9fd75cee3724e6b68cb65beeebaaa38 (commit)
via a879823f631381cefc4458c28f06c36803e30530 (commit)
via 91dc5aafab57c131973244b771392cab083ab469 (commit)
via efa2c6e65983a0917119d7eaf1daa42bc440f52a (commit)
via 5db84e82f71711b6897f06c7da05dc28bba205fa (commit)
via 48ef04b9f09e2f739d3f8c5aba1c7dede305d416 (commit)
via 9379719d46bb351f87aa9c32a0325a65900b9faa (commit)
via ba6e96dfc00cc6ce4f6ab299bdbff616a523ce90 (commit)
via 79a2d819d596e610f26c08beee53f5432bfbb360 (commit)
via 9ec15072e6631578584bd9c08d26025e807a8d48 (commit)
via f9a3dea1aed09b7b1d885c3116c63310e0440007 (commit)
via 40d5d40955a88dbd5cd7c25292268f1ab4536bda (commit)
via 667a7121e08d4fffc24cafdc3ed474374782b959 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (08edbfda915126a1e142fd72d81be17318e59889)
\
N -- N -- N (4ce855e6b1368cc6ce95c27da2238fa39060c069)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4ce855e6b1368cc6ce95c27da2238fa39060c069
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 13 17:13:42 2017 -0500
8311: Try some git_tree mounts in TestSetupMounts.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index e1d9fed..4d76e34 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -28,6 +28,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/manifest"
dockertypes "github.com/docker/docker/api/types"
@@ -1263,6 +1264,64 @@ func (s *TestSuite) TestSetupMounts(c *C) {
cr.CleanupDirs()
checkEmpty()
}
+
+ // git_tree mounts
+ {
+ i = 0
+ cr.ArvMountPoint = ""
+ (*GitMountSuite)(nil).useTestGitServer(c)
+ cr.Container.Mounts = make(map[string]arvados.Mount)
+ cr.Container.Mounts = map[string]arvados.Mount{
+ "/tip": {
+ Kind: "git_tree",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
+ Path: "/",
+ },
+ "/non-tip": {
+ Kind: "git_tree",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ Path: "/",
+ Writable: true,
+ },
+ }
+ cr.OutputPath = "/tmp"
+
+ err := cr.SetupMounts()
+ c.Check(err, IsNil)
+
+ // dirMap[mountpoint] == tmpdir
+ dirMap := make(map[string]string)
+ for _, bind := range cr.Binds {
+ tokens := strings.Split(bind, ":")
+ dirMap[tokens[1]] = tokens[0]
+
+ if cr.Container.Mounts[tokens[1]].Writable {
+ c.Check(len(tokens), Equals, 2)
+ } else {
+ c.Check(len(tokens), Equals, 3)
+ c.Check(tokens[2], Equals, "ro")
+ }
+ }
+
+ data, err := ioutil.ReadFile(dirMap["/tip"] + "/dir1/dir2/file with mode 0644")
+ c.Check(err, IsNil)
+ c.Check(string(data), Equals, "\000\001\002\003")
+ _, err = ioutil.ReadFile(dirMap["/tip"] + "/file only on testbranch")
+ c.Check(err, FitsTypeOf, &os.PathError{})
+ c.Check(os.IsNotExist(err), Equals, true)
+
+ data, err = ioutil.ReadFile(dirMap["/non-tip"] + "/dir1/dir2/file with mode 0644")
+ c.Check(err, IsNil)
+ c.Check(string(data), Equals, "\000\001\002\003")
+ data, err = ioutil.ReadFile(dirMap["/non-tip"] + "/file only on testbranch")
+ c.Check(err, IsNil)
+ c.Check(string(data), Equals, "testfile\n")
+
+ cr.CleanupDirs()
+ checkEmpty()
+ }
}
func (s *TestSuite) TestStdout(c *C) {
diff --git a/services/crunch-run/git_mount.go b/services/crunch-run/git_mount.go
index a792741..110a81b 100644
--- a/services/crunch-run/git_mount.go
+++ b/services/crunch-run/git_mount.go
@@ -33,7 +33,10 @@ func (gm gitMount) extractTree(ac IArvadosClient, dir string) error {
baseURL, err := ac.Discovery("gitUrl")
if err != nil {
return fmt.Errorf("discover gitUrl from API: %s", err)
+ } else if _, ok := baseURL.(string); !ok {
+ return fmt.Errorf("discover gitUrl from API: expected string, found %T", baseURL)
}
+
u, err := url.Parse(baseURL.(string))
if err != nil {
return fmt.Errorf("parse gitUrl %q: %s", baseURL, err)
diff --git a/services/crunch-run/git_mount_test.go b/services/crunch-run/git_mount_test.go
index 4c6ce7f..e984000 100644
--- a/services/crunch-run/git_mount_test.go
+++ b/services/crunch-run/git_mount_test.go
@@ -22,15 +22,10 @@ type GitMountSuite struct {
var _ = check.Suite(&GitMountSuite{})
-func (s *GitMountSuite) SetUpSuite(c *check.C) {
- git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
-}
-
func (s *GitMountSuite) SetUpTest(c *check.C) {
- port, err := ioutil.ReadFile("../../tmp/arv-git-httpd-ssl.port")
- c.Assert(err, check.IsNil)
- discoveryMap["gitUrl"] = "https://localhost:" + string(port)
+ s.useTestGitServer(c)
+ var err error
s.tmpdir, err = ioutil.TempDir("", "")
c.Assert(err, check.IsNil)
}
@@ -117,6 +112,17 @@ func (s *GitMountSuite) TestNonexistentCommit(c *check.C) {
s.checkTmpdirContents(c, []string{})
}
+func (s *GitMountSuite) TestGitUrlDiscoveryFails(c *check.C) {
+ delete(discoveryMap, "gitUrl")
+ gm := gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir)
+ c.Check(err, check.ErrorMatches, ".*gitUrl.*")
+}
+
func (s *GitMountSuite) TestInvalid(c *check.C) {
for _, trial := range []struct {
gm gitMount
@@ -150,7 +156,7 @@ func (s *GitMountSuite) TestInvalid(c *check.C) {
c.Check(err, check.NotNil)
s.checkTmpdirContents(c, []string{})
- err := trial.gm.validate()
+ err = trial.gm.validate()
c.Check(err, check.ErrorMatches, trial.matcher)
}
}
@@ -162,3 +168,11 @@ func (s *GitMountSuite) checkTmpdirContents(c *check.C, expect []string) {
c.Check(err, check.IsNil)
c.Check(names, check.DeepEquals, expect)
}
+
+func (*GitMountSuite) useTestGitServer(c *check.C) {
+ git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
+
+ port, err := ioutil.ReadFile("../../tmp/arv-git-httpd-ssl.port")
+ c.Assert(err, check.IsNil)
+ discoveryMap["gitUrl"] = "https://localhost:" + string(port)
+}
commit 4343e1f1035baaa22f117d119c6e6bc35b1e385b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Dec 12 18:56:07 2017 -0500
8311: Add git_tree mount type.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 7d1d4c9..45f603d 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -323,6 +323,9 @@ start_api() {
}
start_nginx_proxy_services() {
+ if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
+ return
+ fi
echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
cd "$WORKSPACE" \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
@@ -822,6 +825,13 @@ install_apiserver() {
fi
cd "$WORKSPACE/services/api" \
+ && rm -rf tmp/git \
+ && mkdir -p tmp/git \
+ && cd tmp/git \
+ && tar xf ../../test/test.git.tar \
+ || return 1
+
+ cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle exec rake db:drop \
&& RAILS_ENV=test bundle exec rake db:setup \
&& RAILS_ENV=test bundle exec rake db:fixtures:load
@@ -904,7 +914,7 @@ if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
exit_cleanly
fi
-start_api || { stop_services; fatal "start_api"; }
+start_api && start_nginx_proxy_services || { stop_services; fatal "start_api"; }
test_ruby_sdk() {
cd "$WORKSPACE/sdk/ruby" \
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index a38d95c..24f3faa 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -245,6 +245,7 @@ type DiscoveryDocument struct {
BasePath string `json:"basePath"`
DefaultCollectionReplication int `json:"defaultCollectionReplication"`
BlobSignatureTTL int64 `json:"blobSignatureTtl"`
+ GitURL string `json:"gitUrl"`
Schemas map[string]Schema `json:"schemas"`
Resources map[string]Resource `json:"resources"`
}
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index 7e588be..16726b7 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -32,6 +32,7 @@ type Mount struct {
Content interface{} `json:"content"`
ExcludeFromOutput bool `json:"exclude_from_output"`
Capacity int64 `json:"capacity"`
+ Commit string `json:"commit"` // only if kind=="git_tree"
}
// RuntimeConstraints specify a container's compute resources (RAM,
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index a237829..f413755 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -60,6 +60,7 @@ class Arvados::V1::SchemaController < ApplicationController
websocketUrl: Rails.application.config.websocket_address,
workbenchUrl: Rails.application.config.workbench_address,
keepWebServiceUrl: Rails.application.config.keep_web_service_url,
+ gitUrl: Rails.application.config.git_repo_https_base,
parameters: {
alt: {
type: "string",
diff --git a/services/api/test/test.git.tar b/services/api/test/test.git.tar
index faa0d65..93cff81 100644
Binary files a/services/api/test/test.git.tar and b/services/api/test/test.git.tar differ
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index f3f754b..d262b3d 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -538,6 +538,22 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
return fmt.Errorf("writing temp file: %v", err)
}
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind))
+
+ case mnt.Kind == "git_tree":
+ tmpdir, err := runner.MkTempDir("", "")
+ if err != nil {
+ return fmt.Errorf("creating temp dir: %v", err)
+ }
+ runner.CleanupTempDir = append(runner.CleanupTempDir, tmpdir)
+ err = gitMount(mnt).extractTree(runner.ArvClient, tmpdir)
+ if err != nil {
+ return err
+ }
+ bind := tmpdir + ":" + bind
+ if !mnt.Writable {
+ bind = bind + ":ro"
+ }
+ runner.Binds = append(runner.Binds, bind)
}
}
diff --git a/services/crunch-run/git_mount.go b/services/crunch-run/git_mount.go
new file mode 100644
index 0000000..a792741
--- /dev/null
+++ b/services/crunch-run/git_mount.go
@@ -0,0 +1,75 @@
+package main
+
+import (
+ "fmt"
+ "net/url"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "gopkg.in/src-d/go-billy.v3/osfs"
+ git "gopkg.in/src-d/go-git.v4"
+ git_config "gopkg.in/src-d/go-git.v4/config"
+ git_plumbing "gopkg.in/src-d/go-git.v4/plumbing"
+ git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+ "gopkg.in/src-d/go-git.v4/storage/memory"
+)
+
+type gitMount arvados.Mount
+
+func (gm gitMount) validate() error {
+ if gm.Path != "/" {
+ return fmt.Errorf("cannot mount git_tree path %q -- only \"/\" is supported", gm.Path)
+ }
+ return nil
+}
+
+// ExtractTree extracts the specified tree into dir, which is an
+// existing empty local directory.
+func (gm gitMount) extractTree(ac IArvadosClient, dir string) error {
+ err := gm.validate()
+ if err != nil {
+ return err
+ }
+ baseURL, err := ac.Discovery("gitUrl")
+ if err != nil {
+ return fmt.Errorf("discover gitUrl from API: %s", err)
+ }
+ u, err := url.Parse(baseURL.(string))
+ if err != nil {
+ return fmt.Errorf("parse gitUrl %q: %s", baseURL, err)
+ }
+ u, err = u.Parse("/" + gm.UUID + ".git")
+ if err != nil {
+ return fmt.Errorf("build git url from %q, %q: %s", baseURL, gm.UUID, err)
+ }
+ store := memory.NewStorage()
+ repo, err := git.Init(store, osfs.New(dir))
+ if err != nil {
+ return fmt.Errorf("init repo: %s", err)
+ }
+ _, err = repo.CreateRemote(&git_config.RemoteConfig{
+ Name: "origin",
+ URLs: []string{u.String()},
+ })
+ if err != nil {
+ return fmt.Errorf("create remote %q: %s", u.String(), err)
+ }
+ err = repo.Fetch(&git.FetchOptions{
+ RemoteName: "origin",
+ Auth: git_http.NewBasicAuth("none", arvadostest.ActiveToken),
+ })
+ if err != nil {
+ return fmt.Errorf("git fetch %q: %s", u.String(), err)
+ }
+ wt, err := repo.Worktree()
+ if err != nil {
+ return fmt.Errorf("worktree failed: %s", err)
+ }
+ err = wt.Checkout(&git.CheckoutOptions{
+ Hash: git_plumbing.NewHash(gm.Commit),
+ })
+ if err != nil {
+ return fmt.Errorf("checkout failed: %s", err)
+ }
+ return nil
+}
diff --git a/services/crunch-run/git_mount_test.go b/services/crunch-run/git_mount_test.go
new file mode 100644
index 0000000..4c6ce7f
--- /dev/null
+++ b/services/crunch-run/git_mount_test.go
@@ -0,0 +1,164 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+ git_client "gopkg.in/src-d/go-git.v4/plumbing/transport/client"
+ git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+)
+
+type GitMountSuite struct {
+ tmpdir string
+}
+
+var _ = check.Suite(&GitMountSuite{})
+
+func (s *GitMountSuite) SetUpSuite(c *check.C) {
+ git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
+}
+
+func (s *GitMountSuite) SetUpTest(c *check.C) {
+ port, err := ioutil.ReadFile("../../tmp/arv-git-httpd-ssl.port")
+ c.Assert(err, check.IsNil)
+ discoveryMap["gitUrl"] = "https://localhost:" + string(port)
+
+ s.tmpdir, err = ioutil.TempDir("", "")
+ c.Assert(err, check.IsNil)
+}
+
+func (s *GitMountSuite) TearDownTest(c *check.C) {
+ err := os.RemoveAll(s.tmpdir)
+ c.Check(err, check.IsNil)
+}
+
+// Commit fd3531f is crunch-run-tree-test
+func (s *GitMountSuite) TestextractTree(c *check.C) {
+ gm := gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir)
+ c.Check(err, check.IsNil)
+
+ fnm := filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0644")
+ data, err := ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(data, check.DeepEquals, []byte{0, 1, 2, 3})
+ fi, err := os.Stat(fnm)
+ c.Check(err, check.IsNil)
+ if err == nil {
+ c.Check(fi.Mode(), check.Equals, os.FileMode(0644))
+ }
+
+ fnm = filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0755")
+ data, err = ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(string(data), check.DeepEquals, "#!/bin/sh\nexec echo OK\n")
+ fi, err = os.Stat(fnm)
+ c.Check(err, check.IsNil)
+ if err == nil {
+ c.Check(fi.Mode(), check.Equals, os.FileMode(0755))
+ }
+
+ // Ensure there's no extra stuff like a ".git" dir
+ s.checkTmpdirContents(c, []string{"dir1"})
+}
+
+// Commit 5ebfab0 is not the tip of any branch or tag, but is
+// reachable in branch "crunch-run-non-tip-test".
+func (s *GitMountSuite) TestExtractNonTipCommit(c *check.C) {
+ gm := gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir)
+ c.Check(err, check.IsNil)
+
+ fnm := filepath.Join(s.tmpdir, "file only on testbranch")
+ data, err := ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(string(data), check.DeepEquals, "testfile\n")
+}
+
+func (s *GitMountSuite) TestNonexistentRepository(c *check.C) {
+ gm := gitMount{
+ Path: "/",
+ UUID: "zzzzz-s0uqq-nonexistentrepo",
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir)
+ c.Check(err, check.NotNil)
+ c.Check(err, check.ErrorMatches, ".*repository not found.*")
+
+ s.checkTmpdirContents(c, []string{})
+}
+
+func (s *GitMountSuite) TestNonexistentCommit(c *check.C) {
+ gm := gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "bb66b6bb6b6bbb6b6b6b66b6b6b6b6b6b6b6b66b",
+ }
+ err := gm.extractTree(&ArvTestClient{}, s.tmpdir)
+ c.Check(err, check.NotNil)
+ c.Check(err, check.ErrorMatches, ".*object not found.*")
+
+ s.checkTmpdirContents(c, []string{})
+}
+
+func (s *GitMountSuite) TestInvalid(c *check.C) {
+ for _, trial := range []struct {
+ gm gitMount
+ matcher string
+ }{
+ {
+ gm: gitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "abc123",
+ },
+ matcher: ".*sha1.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/dir1/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ },
+ matcher: ".*path.*",
+ },
+ {
+ gm: gitMount{
+ Path: "/",
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ },
+ matcher: ".*UUID.*",
+ },
+ } {
+ err := trial.gm.extractTree(&ArvTestClient{}, s.tmpdir)
+ c.Check(err, check.NotNil)
+ s.checkTmpdirContents(c, []string{})
+
+ err := trial.gm.validate()
+ c.Check(err, check.ErrorMatches, trial.matcher)
+ }
+}
+
+func (s *GitMountSuite) checkTmpdirContents(c *check.C, expect []string) {
+ f, err := os.Open(s.tmpdir)
+ c.Check(err, check.IsNil)
+ names, err := f.Readdirnames(-1)
+ c.Check(err, check.IsNil)
+ c.Check(names, check.DeepEquals, expect)
+}
commit a6c6892623dcd1ca882478a06f274a19d7951cb8
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 14 10:29:10 2017 -0500
8311: Add auth handler tests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/arv-git-httpd/auth_handler_test.go b/services/arv-git-httpd/auth_handler_test.go
index df64999..05fde03 100644
--- a/services/arv-git-httpd/auth_handler_test.go
+++ b/services/arv-git-httpd/auth_handler_test.go
@@ -5,10 +5,16 @@
package main
import (
+ "io"
+ "log"
"net/http"
"net/http/httptest"
"net/url"
+ "path/filepath"
+ "strings"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
check "gopkg.in/check.v1"
)
@@ -16,6 +22,116 @@ var _ = check.Suite(&AuthHandlerSuite{})
type AuthHandlerSuite struct{}
+func (s *AuthHandlerSuite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+}
+
+func (s *AuthHandlerSuite) TearDownSuite(c *check.C) {
+ arvadostest.StopAPI()
+}
+
+func (s *AuthHandlerSuite) SetUpTest(c *check.C) {
+ arvadostest.ResetEnv()
+ repoRoot, err := filepath.Abs("../api/tmp/git/test")
+ c.Assert(err, check.IsNil)
+ theConfig = &Config{
+ Client: arvados.Client{
+ APIHost: arvadostest.APIHost(),
+ Insecure: true,
+ },
+ Listen: ":0",
+ GitCommand: "/usr/bin/git",
+ RepoRoot: repoRoot,
+ ManagementToken: arvadostest.ManagementToken,
+ }
+}
+
+func (s *AuthHandlerSuite) TestPermission(c *check.C) {
+ h := &authHandler{handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ log.Printf("%v", r.URL)
+ io.WriteString(w, r.URL.Path)
+ })}
+ baseURL, err := url.Parse("http://git.example/")
+ c.Assert(err, check.IsNil)
+ for _, trial := range []struct {
+ label string
+ token string
+ pathIn string
+ pathOut string
+ status int
+ }{
+ {
+ label: "read repo by name",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2Name + ".git/git-upload-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-upload-pack",
+ },
+ {
+ label: "read repo by uuid",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2UUID + ".git/git-upload-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-upload-pack",
+ },
+ {
+ label: "write repo by name",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2Name + ".git/git-receive-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-receive-pack",
+ },
+ {
+ label: "write repo by uuid",
+ token: arvadostest.ActiveToken,
+ pathIn: arvadostest.Repository2UUID + ".git/git-receive-pack",
+ pathOut: arvadostest.Repository2UUID + ".git/git-receive-pack",
+ },
+ {
+ label: "uuid not found",
+ token: arvadostest.ActiveToken,
+ pathIn: strings.Replace(arvadostest.Repository2UUID, "6", "z", -1) + ".git/git-upload-pack",
+ status: http.StatusNotFound,
+ },
+ {
+ label: "name not found",
+ token: arvadostest.ActiveToken,
+ pathIn: "nonexistent-bogus.git/git-upload-pack",
+ status: http.StatusNotFound,
+ },
+ {
+ label: "read read-only repo",
+ token: arvadostest.SpectatorToken,
+ pathIn: arvadostest.FooRepoName + ".git/git-upload-pack",
+ pathOut: arvadostest.FooRepoUUID + "/.git/git-upload-pack",
+ },
+ {
+ label: "write read-only repo",
+ token: arvadostest.SpectatorToken,
+ pathIn: arvadostest.FooRepoName + ".git/git-receive-pack",
+ status: http.StatusForbidden,
+ },
+ } {
+ c.Logf("trial label: %q", trial.label)
+ u, err := baseURL.Parse(trial.pathIn)
+ c.Assert(err, check.IsNil)
+ resp := httptest.NewRecorder()
+ req := &http.Request{
+ Method: "POST",
+ URL: u,
+ Header: http.Header{
+ "Authorization": {"Bearer " + trial.token}}}
+ h.ServeHTTP(resp, req)
+ if trial.status == 0 {
+ trial.status = http.StatusOK
+ }
+ c.Check(resp.Code, check.Equals, trial.status)
+ if trial.status < 400 {
+ if trial.pathOut != "" && !strings.HasPrefix(trial.pathOut, "/") {
+ trial.pathOut = "/" + trial.pathOut
+ }
+ c.Check(resp.Body.String(), check.Equals, trial.pathOut)
+ }
+ }
+}
+
func (s *AuthHandlerSuite) TestCORS(c *check.C) {
h := &authHandler{}
commit 1d2020d448c45a1bb8a7b1f6aa04ed1d87dc353b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Dec 12 18:58:15 2017 -0500
8311: Access git repos by UUID.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 7858fa0..eab7a27 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -30,6 +30,13 @@ const (
Dispatch1AuthUUID = "zzzzz-gj3su-k9dvestay1plssr"
QueuedContainerUUID = "zzzzz-dz642-queuedcontainer"
+
+ ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
+ ArvadosRepoName = "arvados"
+ FooRepoUUID = "zzzzz-s0uqq-382brsig8rp3666"
+ FooRepoName = "active/foo"
+ Repository2UUID = "zzzzz-s0uqq-382brsig8rp3667"
+ Repository2Name = "active/foo2"
)
// PathologicalManifest : A valid manifest designed to test
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index e940c13..b4dc58b 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -5,9 +5,11 @@
package main
import (
+ "errors"
"log"
"net/http"
"os"
+ "regexp"
"strings"
"sync"
"time"
@@ -118,27 +120,17 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// Ask API server whether the repository is readable using
// this token (by trying to read it!)
arv.ApiToken = apiToken
- reposFound := arvadosclient.Dict{}
- if err := arv.List("repositories", arvadosclient.Dict{
- "filters": [][]string{{"name", "=", repoName}},
- }, &reposFound); err != nil {
+ repoUUID, err := h.lookupRepo(arv, repoName)
+ if err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
validApiToken = true
- if avail, ok := reposFound["items_available"].(float64); !ok {
- statusCode, statusText = http.StatusInternalServerError, "bad list response from API"
- return
- } else if avail < 1 {
+ if repoUUID == "" {
statusCode, statusText = http.StatusNotFound, "not found"
return
- } else if avail > 1 {
- statusCode, statusText = http.StatusInternalServerError, "name collision"
- return
}
- repoUUID := reposFound["items"].([]interface{})[0].(map[string]interface{})["uuid"].(string)
-
isWrite := strings.HasSuffix(r.URL.Path, "/git-receive-pack")
if !isWrite {
statusText = "read"
@@ -191,3 +183,28 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
h.handler.ServeHTTP(w, r)
}
+
+var uuidRegexp = regexp.MustCompile(`^[0-9a-z]{5}-s0uqq-[0-9a-z]{15}$`)
+
+func (h *authHandler) lookupRepo(arv *arvadosclient.ArvadosClient, repoName string) (string, error) {
+ reposFound := arvadosclient.Dict{}
+ var column string
+ if uuidRegexp.MatchString(repoName) {
+ column = "uuid"
+ } else {
+ column = "name"
+ }
+ err := arv.List("repositories", arvadosclient.Dict{
+ "filters": [][]string{{column, "=", repoName}},
+ }, &reposFound)
+ if err != nil {
+ return "", err
+ } else if avail, ok := reposFound["items_available"].(float64); !ok {
+ return "", errors.New("bad list response from API")
+ } else if avail < 1 {
+ return "", nil
+ } else if avail > 1 {
+ return "", errors.New("name collision")
+ }
+ return reposFound["items"].([]interface{})[0].(map[string]interface{})["uuid"].(string), nil
+}
commit 314f9396cd140456a5e5a6d29e4d7741de0181f4
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 14 10:26:08 2017 -0500
8311: Fix "read" or "write" mistakenly appended to response data.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index db37ebb..e940c13 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -70,7 +70,9 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// Nobody has called WriteHeader yet: that
// must be our job.
w.WriteHeader(statusCode)
- w.Write([]byte(statusText))
+ if statusCode >= 400 {
+ w.Write([]byte(statusText))
+ }
}
// If the given password is a valid token, log the first 10 characters of the token.
commit 9dd3035e4c844d2f94aac8e1e2a37f20b42594ea
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Dec 14 10:25:14 2017 -0500
8311: Remove debug printf.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 617c732..db37ebb 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -29,7 +29,6 @@ func (h *authHandler) setup() {
log.Fatal(err)
}
h.clientPool = &arvadosclient.ClientPool{Prototype: ac}
- log.Printf("%+v", h.clientPool.Prototype)
}
func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list