[ARVADOS] created: 1.1.1-231-g00a2666

Git user git at public.curoverse.com
Wed Dec 13 15:53:37 EST 2017


        at  00a2666ce587219f8c984a02cd6c7423638abbfb (commit)


commit 00a2666ce587219f8c984a02cd6c7423638abbfb
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Tue Dec 12 18:56:07 2017 -0500

    8311: Add git_tree mount type.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 7d1d4c9..45f603d 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -323,6 +323,9 @@ start_api() {
 }
 
 start_nginx_proxy_services() {
+    if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
+        return
+    fi
     echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
     cd "$WORKSPACE" \
         && python sdk/python/tests/run_test_server.py start_keep_proxy \
@@ -822,6 +825,13 @@ install_apiserver() {
     fi
 
     cd "$WORKSPACE/services/api" \
+        && rm -rf tmp/git \
+        && mkdir -p tmp/git \
+        && cd tmp/git \
+        && tar xf ../../test/test.git.tar \
+            || return 1
+
+    cd "$WORKSPACE/services/api" \
         && RAILS_ENV=test bundle exec rake db:drop \
         && RAILS_ENV=test bundle exec rake db:setup \
         && RAILS_ENV=test bundle exec rake db:fixtures:load
@@ -904,7 +914,7 @@ if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
   exit_cleanly
 fi
 
-start_api || { stop_services; fatal "start_api"; }
+start_api && start_nginx_proxy_services || { stop_services; fatal "start_api"; }
 
 test_ruby_sdk() {
     cd "$WORKSPACE/sdk/ruby" \
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index a38d95c..24f3faa 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -245,6 +245,7 @@ type DiscoveryDocument struct {
 	BasePath                     string              `json:"basePath"`
 	DefaultCollectionReplication int                 `json:"defaultCollectionReplication"`
 	BlobSignatureTTL             int64               `json:"blobSignatureTtl"`
+	GitURL                       string              `json:"gitUrl"`
 	Schemas                      map[string]Schema   `json:"schemas"`
 	Resources                    map[string]Resource `json:"resources"`
 }
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index 7e588be..16726b7 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -32,6 +32,7 @@ type Mount struct {
 	Content           interface{} `json:"content"`
 	ExcludeFromOutput bool        `json:"exclude_from_output"`
 	Capacity          int64       `json:"capacity"`
+	Commit            string      `json:"commit"` // only if kind=="git_tree"
 }
 
 // RuntimeConstraints specify a container's compute resources (RAM,
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 7858fa0..f8fa267 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -30,6 +30,13 @@ const (
 	Dispatch1AuthUUID = "zzzzz-gj3su-k9dvestay1plssr"
 
 	QueuedContainerUUID = "zzzzz-dz642-queuedcontainer"
+
+	ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
+	ArvadosRepoName = "arvados"
+	FooRepoUUID     = "zzzzz-s0uqq-382brsig8rp3666"
+	FooRepoName     = "active/foo"
+	Repository2UUID = "zzzzz-s0uqq-382brsig8rp3667"
+	Repository2Name = "repository2"
 )
 
 // PathologicalManifest : A valid manifest designed to test
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index a237829..f413755 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -60,6 +60,7 @@ class Arvados::V1::SchemaController < ApplicationController
         websocketUrl: Rails.application.config.websocket_address,
         workbenchUrl: Rails.application.config.workbench_address,
         keepWebServiceUrl: Rails.application.config.keep_web_service_url,
+        gitUrl: Rails.application.config.git_repo_https_base,
         parameters: {
           alt: {
             type: "string",
diff --git a/services/api/test/test.git.tar b/services/api/test/test.git.tar
index faa0d65..93cff81 100644
Binary files a/services/api/test/test.git.tar and b/services/api/test/test.git.tar differ
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index f3f754b..3cddbad 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -538,6 +538,22 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
 				return fmt.Errorf("writing temp file: %v", err)
 			}
 			runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind))
+
+		case mnt.Kind == "git_tree":
+			tmpdir, err := runner.MkTempDir("", "")
+			if err != nil {
+				return fmt.Errorf("creating temp dir: %v", err)
+			}
+			runner.CleanupTempDir = append(runner.CleanupTempDir, tmpdir)
+			err = GitMount(mnt).ExtractTree(runner.ArvClient, tmpdir)
+			if err != nil {
+				return err
+			}
+			bind := tmpdir + ":" + bind
+			if !mnt.Writable {
+				bind = bind + ":ro"
+			}
+			runner.Binds = append(runner.Binds, bind)
 		}
 	}
 
diff --git a/services/crunch-run/git_mount.go b/services/crunch-run/git_mount.go
new file mode 100644
index 0000000..b73bdc7
--- /dev/null
+++ b/services/crunch-run/git_mount.go
@@ -0,0 +1,67 @@
+package main
+
+import (
+	"fmt"
+	"net/url"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	"gopkg.in/src-d/go-billy.v3/osfs"
+	git "gopkg.in/src-d/go-git.v4"
+	git_config "gopkg.in/src-d/go-git.v4/config"
+	git_plumbing "gopkg.in/src-d/go-git.v4/plumbing"
+	git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+	"gopkg.in/src-d/go-git.v4/storage/memory"
+)
+
+type GitMount arvados.Mount
+
+// ExtractTree extracts the specified tree into dir, which is an
+// existing empty local directory.
+func (gm GitMount) ExtractTree(ac IArvadosClient, dir string) error {
+	if gm.Path != "/" {
+		return fmt.Errorf("cannot mount git_tree path %q -- only \"/\" is supported", gm.Path)
+	}
+	baseURL, err := ac.Discovery("gitUrl")
+	if err != nil {
+		return fmt.Errorf("discover gitUrl from API: %s", err)
+	}
+	u, err := url.Parse(baseURL.(string))
+	if err != nil {
+		return fmt.Errorf("parse gitUrl %q: %s", baseURL, err)
+	}
+	u, err = u.Parse("/" + gm.UUID + ".git")
+	if err != nil {
+		return fmt.Errorf("build git url from %q, %q: %s", baseURL, gm.UUID, err)
+	}
+	store := memory.NewStorage()
+	repo, err := git.Init(store, osfs.New(dir))
+	if err != nil {
+		return fmt.Errorf("init repo: %s", err)
+	}
+	_, err = repo.CreateRemote(&git_config.RemoteConfig{
+		Name: "origin",
+		URLs: []string{u.String()},
+	})
+	if err != nil {
+		return fmt.Errorf("create remote %q: %s", u.String(), err)
+	}
+	err = repo.Fetch(&git.FetchOptions{
+		RemoteName: "origin",
+		Auth:       git_http.NewBasicAuth("none", arvadostest.ActiveToken),
+	})
+	if err != nil {
+		return fmt.Errorf("git fetch %q: %s", u.String(), err)
+	}
+	wt, err := repo.Worktree()
+	if err != nil {
+		return fmt.Errorf("worktree failed: %s", err)
+	}
+	err = wt.Checkout(&git.CheckoutOptions{
+		Hash: git_plumbing.NewHash(gm.Commit),
+	})
+	if err != nil {
+		return fmt.Errorf("checkout failed: %s", err)
+	}
+	return nil
+}
diff --git a/services/crunch-run/git_mount_test.go b/services/crunch-run/git_mount_test.go
new file mode 100644
index 0000000..4c4cfe3
--- /dev/null
+++ b/services/crunch-run/git_mount_test.go
@@ -0,0 +1,126 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	check "gopkg.in/check.v1"
+	git_client "gopkg.in/src-d/go-git.v4/plumbing/transport/client"
+	git_http "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+)
+
+type GitMountSuite struct {
+	tmpdir string
+}
+
+var _ = check.Suite(&GitMountSuite{})
+
+func (s *GitMountSuite) SetUpSuite(c *check.C) {
+	git_client.InstallProtocol("https", git_http.NewClient(arvados.InsecureHTTPClient))
+}
+
+func (s *GitMountSuite) SetUpTest(c *check.C) {
+	port, err := ioutil.ReadFile("../../tmp/arv-git-httpd-ssl.port")
+	c.Assert(err, check.IsNil)
+	discoveryMap["gitUrl"] = "https://localhost:" + string(port)
+
+	s.tmpdir, err = ioutil.TempDir("", "")
+	c.Assert(err, check.IsNil)
+}
+
+func (s *GitMountSuite) TearDownTest(c *check.C) {
+	err := os.RemoveAll(s.tmpdir)
+	c.Check(err, check.IsNil)
+}
+
+// Commit fd3531f is crunch-run-tree-test
+func (s *GitMountSuite) TestExtractTree(c *check.C) {
+	gm := GitMount{
+		Path:   "/",
+		UUID:   arvadostest.Repository2UUID,
+		Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
+	}
+	err := gm.ExtractTree(&ArvTestClient{}, s.tmpdir)
+	c.Check(err, check.IsNil)
+
+	fnm := filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0644")
+	data, err := ioutil.ReadFile(fnm)
+	c.Check(err, check.IsNil)
+	c.Check(data, check.DeepEquals, []byte{0, 1, 2, 3})
+	fi, err := os.Stat(fnm)
+	c.Check(err, check.IsNil)
+	if err == nil {
+		c.Check(fi.Mode(), check.Equals, os.FileMode(0644))
+	}
+
+	fnm = filepath.Join(s.tmpdir, "dir1/dir2/file with mode 0755")
+	data, err = ioutil.ReadFile(fnm)
+	c.Check(err, check.IsNil)
+	c.Check(string(data), check.DeepEquals, "#!/bin/sh\nexec echo OK\n")
+	fi, err = os.Stat(fnm)
+	c.Check(err, check.IsNil)
+	if err == nil {
+		c.Check(fi.Mode(), check.Equals, os.FileMode(0755))
+	}
+
+	// Ensure there's no extra stuff like a ".git" dir
+	s.checkTmpdirContents(c, []string{"dir1"})
+}
+
+// Commit 5ebfab0 is not the tip of any branch or tag, but is
+// reachable in branch "crunch-run-non-tip-test".
+func (s *GitMountSuite) TestExtractNonTipCommit(c *check.C) {
+	gm := GitMount{
+		Path:   "/",
+		UUID:   arvadostest.Repository2UUID,
+		Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+	}
+	err := gm.ExtractTree(&ArvTestClient{}, s.tmpdir)
+	c.Check(err, check.IsNil)
+
+	fnm := filepath.Join(s.tmpdir, "file only on testbranch")
+	data, err := ioutil.ReadFile(fnm)
+	c.Check(err, check.IsNil)
+	c.Check(string(data), check.DeepEquals, "testfile\n")
+}
+
+func (s *GitMountSuite) TestNonexistentRepository(c *check.C) {
+	gm := GitMount{
+		Path:   "/",
+		UUID:   "zzzzz-s0uqq-nonexistentrepo",
+		Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
+	}
+	err := gm.ExtractTree(&ArvTestClient{}, s.tmpdir)
+	c.Check(err, check.NotNil)
+	c.Check(err, check.ErrorMatches, ".*repository not found.*")
+
+	s.checkTmpdirContents(c, []string{})
+}
+
+func (s *GitMountSuite) TestNonexistentCommit(c *check.C) {
+	gm := GitMount{
+		Path:   "/",
+		UUID:   arvadostest.Repository2UUID,
+		Commit: "bb66b6bb6b6bbb6b6b6b66b6b6b6b6b6b6b6b66b",
+	}
+	err := gm.ExtractTree(&ArvTestClient{}, s.tmpdir)
+	c.Check(err, check.NotNil)
+	c.Check(err, check.ErrorMatches, ".*object not found.*")
+
+	s.checkTmpdirContents(c, []string{})
+}
+
+func (s *GitMountSuite) checkTmpdirContents(c *check.C, expect []string) {
+	f, err := os.Open(s.tmpdir)
+	c.Check(err, check.IsNil)
+	names, err := f.Readdirnames(-1)
+	c.Check(err, check.IsNil)
+	c.Check(names, check.DeepEquals, expect)
+}

commit d98fbd3931822993fd80ef605ed9d14d777274b1
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Tue Dec 12 18:58:15 2017 -0500

    8311: Access git repos by UUID.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 617c732..e162230 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -119,11 +119,23 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
 	arv.ApiToken = apiToken
 	reposFound := arvadosclient.Dict{}
 	if err := arv.List("repositories", arvadosclient.Dict{
-		"filters": [][]string{{"name", "=", repoName}},
+		"filters": [][]string{{"uuid", "=", repoName}},
 	}, &reposFound); err != nil {
 		statusCode, statusText = http.StatusInternalServerError, err.Error()
 		return
 	}
+	if avail, ok := reposFound["items_available"].(float64); !ok {
+		statusCode, statusText = http.StatusInternalServerError, "bad list response from API"
+		return
+	} else if avail < 1 {
+		reposFound = arvadosclient.Dict{}
+		if err := arv.List("repositories", arvadosclient.Dict{
+			"filters": [][]string{{"name", "=", repoName}},
+		}, &reposFound); err != nil {
+			statusCode, statusText = http.StatusInternalServerError, err.Error()
+			return
+		}
+	}
 	validApiToken = true
 	if avail, ok := reposFound["items_available"].(float64); !ok {
 		statusCode, statusText = http.StatusInternalServerError, "bad list response from API"

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list