[ARVADOS] created: 1.1.1-231-g00a2666
Git user
git at public.curoverse.com
Wed Dec 13 15:53:37 EST 2017
at 00a2666ce587219f8c984a02cd6c7423638abbfb (commit)
commit 00a2666ce587219f8c984a02cd6c7423638abbfb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Dec 12 18:56:07 2017 -0500
8311: Add git_tree mount type.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 7d1d4c9..45f603d 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -323,6 +323,9 @@ start_api() {
}
start_nginx_proxy_services() {
+ if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
+ return
+ fi
echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
cd "$WORKSPACE" \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
@@ -822,6 +825,13 @@ install_apiserver() {
fi
cd "$WORKSPACE/services/api" \
+ && rm -rf tmp/git \
+ && mkdir -p tmp/git \
+ && cd tmp/git \
+ && tar xf ../../test/test.git.tar \
+ || return 1
+
+ cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle exec rake db:drop \
&& RAILS_ENV=test bundle exec rake db:setup \
&& RAILS_ENV=test bundle exec rake db:fixtures:load
@@ -904,7 +914,7 @@ if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
exit_cleanly
fi
-start_api || { stop_services; fatal "start_api"; }
+start_api && start_nginx_proxy_services || { stop_services; fatal "start_api"; }
test_ruby_sdk() {
cd "$WORKSPACE/sdk/ruby" \
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index a38d95c..24f3faa 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -245,6 +245,7 @@ type DiscoveryDocument struct {
BasePath string `json:"basePath"`
DefaultCollectionReplication int `json:"defaultCollectionReplication"`
BlobSignatureTTL int64 `json:"blobSignatureTtl"`
+ GitURL string `json:"gitUrl"`
Schemas map[string]Schema `json:"schemas"`
Resources map[string]Resource `json:"resources"`
}
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index 7e588be..16726b7 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -32,6 +32,7 @@ type Mount struct {
Content interface{} `json:"content"`
ExcludeFromOutput bool `json:"exclude_from_output"`
Capacity int64 `json:"capacity"`
+ Commit string `json:"commit"` // only if kind=="git_tree"
}
// RuntimeConstraints specify a container's compute resources (RAM,
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 7858fa0..f8fa267 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -30,6 +30,13 @@ const (
Dispatch1AuthUUID = "zzzzz-gj3su-k9dvestay1plssr"
QueuedContainerUUID = "zzzzz-dz642-queuedcontainer"
+
+ ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
+ ArvadosRepoName = "arvados"
+ FooRepoUUID = "zzzzz-s0uqq-382brsig8rp3666"
+ FooRepoName = "active/foo"
+ Repository2UUID = "zzzzz-s0uqq-382brsig8rp3667"
+ Repository2Name = "repository2"
)
// PathologicalManifest : A valid manifest designed to test
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index a237829..f413755 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -60,6 +60,7 @@ class Arvados::V1::SchemaController < ApplicationController
websocketUrl: Rails.application.config.websocket_address,
workbenchUrl: Rails.application.config.workbench_address,
keepWebServiceUrl: Rails.application.config.keep_web_service_url,
+ gitUrl: Rails.application.config.git_repo_https_base,
parameters: {
alt: {
type: "string",
diff --git a/services/api/test/test.git.tar b/services/api/test/test.git.tar
index faa0d65..93cff81 100644
Binary files a/services/api/test/test.git.tar and b/services/api/test/test.git.tar differ
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index f3f754b..3cddbad 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -538,6 +538,22 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
return fmt.Errorf("writing temp file: %v", err)
}
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind))
+
+ case mnt.Kind == "git_tree":
+ tmpdir, err := runner.MkTempDir("", "")
+ if err != nil {
+ return fmt.Errorf("creating temp dir: %v", err)
+ }
+ runner.CleanupTempDir = append(runner.CleanupTempDir, tmpdir)
+ err = GitMount(mnt).ExtractTree(runner.ArvClient, tmpdir)
+ if err != nil {
+ return err
+ }
+ bind := tmpdir + ":" + bind
+ if !mnt.Writable {
+ bind = bind + ":ro"
+ }
+ runner.Binds = append(runner.Binds, bind)
}
}
diff --git a/services/crunch-run/git_mount.go b/services/crunch-run/git_mount.go
new file mode 100644
index 0000000..b73bdc7
--- /dev/null
+++ b/services/crunch-run/git_mount.go
@@ -0,0 +1,67 @@
+package main
+
+import (
+ "fmt"
+ "net/url"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "gopkg.in/src-d/go-billy.v3/osfs"
+ git "gopkg.in/src-d/go-git.v4"
+ git_config "gopkg.in/src-d/go-git.v4/config"
+ git_plumbing "gopkg.in/src-d/go-git.v4/plumbing"
+ git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+ "gopkg.in/src-d/go-git.v4/storage/memory"
+)
+
+type GitMount arvados.Mount
+
+// ExtractTree extracts the specified tree into dir, which is an
+// existing empty local directory.
+func (gm GitMount) ExtractTree(ac IArvadosClient, dir string) error {
+ if gm.Path != "/" {
+ return fmt.Errorf("cannot mount git_tree path %q -- only \"/\" is supported", gm.Path)
+ }
+ baseURL, err := ac.Discovery("gitUrl")
+ if err != nil {
+ return fmt.Errorf("discover gitUrl from API: %s", err)
+ }
+ u, err := url.Parse(baseURL.(string))
+ if err != nil {
+ return fmt.Errorf("parse gitUrl %q: %s", baseURL, err)
+ }
+ u, err = u.Parse("/" + gm.UUID + ".git")
+ if err != nil {
+ return fmt.Errorf("build git url from %q, %q: %s", baseURL, gm.UUID, err)
+ }
+ store := memory.NewStorage()
+ repo, err := git.Init(store, osfs.New(dir))
+ if err != nil {
+ return fmt.Errorf("init repo: %s", err)
+ }
+ _, err = repo.CreateRemote(&git_config.RemoteConfig{
+ Name: "origin",
+ URLs: []string{u.String()},
+ })
+ if err != nil {
+ return fmt.Errorf("create remote %q: %s", u.String(), err)
+ }
+ err = repo.Fetch(&git.FetchOptions{
+ RemoteName: "origin",
+ Auth: git_http.NewBasicAuth("none", arvadostest.ActiveToken),
+ })
+ if err != nil {
+ return fmt.Errorf("git fetch %q: %s", u.String(), err)
+ }
+ wt, err := repo.Worktree()
+ if err != nil {
+ return fmt.Errorf("worktree failed: %s", err)
+ }
+ err = wt.Checkout(&git.CheckoutOptions{
+ Hash: git_plumbing.NewHash(gm.Commit),
+ })
+ if err != nil {
+ return fmt.Errorf("checkout failed: %s", err)
+ }
+ return nil
+}
diff --git a/services/crunch-run/git_mount_test.go b/services/crunch-run/git_mount_test.go
new file mode 100644
index 0000000..4c4cfe3
--- /dev/null
+++ b/services/crunch-run/git_mount_test.go
@@ -0,0 +1,126 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+ git_client "gopkg.in/src-d/go-git.v4/plumbing/transport/client"
+ git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+)
+
+type GitMountSuite struct {
+ tmpdir string
+}
+
+var _ = check.Suite(&GitMountSuite{})
+
+func (s *GitMountSuite) SetUpSuite(c *check.C) {
+ git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
+}
+
+func (s *GitMountSuite) SetUpTest(c *check.C) {
+ port, err := ioutil.ReadFile("../../tmp/arv-git-httpd-ssl.port")
+ c.Assert(err, check.IsNil)
+ discoveryMap["gitUrl"] = "https://localhost:" + string(port)
+
+ s.tmpdir, err = ioutil.TempDir("", "")
+ c.Assert(err, check.IsNil)
+}
+
+func (s *GitMountSuite) TearDownTest(c *check.C) {
+ err := os.RemoveAll(s.tmpdir)
+ c.Check(err, check.IsNil)
+}
+
+// Commit fd3531f is crunch-run-tree-test
+func (s *GitMountSuite) TestExtractTree(c *check.C) {
+ gm := GitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
+ }
+ err := gm.ExtractTree(&ArvTestClient{}, s.tmpdir)
+ c.Check(err, check.IsNil)
+
+ fnm := filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0644")
+ data, err := ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(data, check.DeepEquals, []byte{0, 1, 2, 3})
+ fi, err := os.Stat(fnm)
+ c.Check(err, check.IsNil)
+ if err == nil {
+ c.Check(fi.Mode(), check.Equals, os.FileMode(0644))
+ }
+
+ fnm = filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0755")
+ data, err = ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(string(data), check.DeepEquals, "#!/bin/sh\nexec echo OK\n")
+ fi, err = os.Stat(fnm)
+ c.Check(err, check.IsNil)
+ if err == nil {
+ c.Check(fi.Mode(), check.Equals, os.FileMode(0755))
+ }
+
+ // Ensure there's no extra stuff like a ".git" dir
+ s.checkTmpdirContents(c, []string{"dir1"})
+}
+
+// Commit 5ebfab0 is not the tip of any branch or tag, but is
+// reachable in branch "crunch-run-non-tip-test".
+func (s *GitMountSuite) TestExtractNonTipCommit(c *check.C) {
+ gm := GitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.ExtractTree(&ArvTestClient{}, s.tmpdir)
+ c.Check(err, check.IsNil)
+
+ fnm := filepath.Join(s.tmpdir, "file only on testbranch")
+ data, err := ioutil.ReadFile(fnm)
+ c.Check(err, check.IsNil)
+ c.Check(string(data), check.DeepEquals, "testfile\n")
+}
+
+func (s *GitMountSuite) TestNonexistentRepository(c *check.C) {
+ gm := GitMount{
+ Path: "/",
+ UUID: "zzzzz-s0uqq-nonexistentrepo",
+ Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+ }
+ err := gm.ExtractTree(&ArvTestClient{}, s.tmpdir)
+ c.Check(err, check.NotNil)
+ c.Check(err, check.ErrorMatches, ".*repository not found.*")
+
+ s.checkTmpdirContents(c, []string{})
+}
+
+func (s *GitMountSuite) TestNonexistentCommit(c *check.C) {
+ gm := GitMount{
+ Path: "/",
+ UUID: arvadostest.Repository2UUID,
+ Commit: "bb66b6bb6b6bbb6b6b6b66b6b6b6b6b6b6b6b66b",
+ }
+ err := gm.ExtractTree(&ArvTestClient{}, s.tmpdir)
+ c.Check(err, check.NotNil)
+ c.Check(err, check.ErrorMatches, ".*object not found.*")
+
+ s.checkTmpdirContents(c, []string{})
+}
+
+func (s *GitMountSuite) checkTmpdirContents(c *check.C, expect []string) {
+ f, err := os.Open(s.tmpdir)
+ c.Check(err, check.IsNil)
+ names, err := f.Readdirnames(-1)
+ c.Check(err, check.IsNil)
+ c.Check(names, check.DeepEquals, expect)
+}
commit d98fbd3931822993fd80ef605ed9d14d777274b1
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Dec 12 18:58:15 2017 -0500
8311: Access git repos by UUID.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 617c732..e162230 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -119,11 +119,23 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
arv.ApiToken = apiToken
reposFound := arvadosclient.Dict{}
if err := arv.List("repositories", arvadosclient.Dict{
- "filters": [][]string{{"name", "=", repoName}},
+ "filters": [][]string{{"uuid", "=", repoName}},
}, &reposFound); err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
+ if avail, ok := reposFound["items_available"].(float64); !ok {
+ statusCode, statusText = http.StatusInternalServerError, "bad list response from API"
+ return
+ } else if avail < 1 {
+ reposFound = arvadosclient.Dict{}
+ if err := arv.List("repositories", arvadosclient.Dict{
+ "filters": [][]string{{"name", "=", repoName}},
+ }, &reposFound); err != nil {
+ statusCode, statusText = http.StatusInternalServerError, err.Error()
+ return
+ }
+ }
validApiToken = true
if avail, ok := reposFound["items_available"].(float64); !ok {
statusCode, statusText = http.StatusInternalServerError, "bad list response from API"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list