[ARVADOS] created: 1.1.1-231-ga49c0d2
Git user
git at public.curoverse.com
Tue Dec 12 18:58:30 EST 2017
at a49c0d27382f2af5e2cb7999b6b283088ad29e7a (commit)
commit a49c0d27382f2af5e2cb7999b6b283088ad29e7a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Dec 12 18:58:15 2017 -0500
8311: Access git repos by UUID.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index 617c732..e162230 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -119,11 +119,23 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
arv.ApiToken = apiToken
reposFound := arvadosclient.Dict{}
if err := arv.List("repositories", arvadosclient.Dict{
- "filters": [][]string{{"name", "=", repoName}},
+ "filters": [][]string{{"uuid", "=", repoName}},
}, &reposFound); err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
}
+ if avail, ok := reposFound["items_available"].(float64); !ok {
+ statusCode, statusText = http.StatusInternalServerError, "bad list response from API"
+ return
+ } else if avail < 1 {
+ reposFound = arvadosclient.Dict{}
+ if err := arv.List("repositories", arvadosclient.Dict{
+ "filters": [][]string{{"name", "=", repoName}},
+ }, &reposFound); err != nil {
+ statusCode, statusText = http.StatusInternalServerError, err.Error()
+ return
+ }
+ }
validApiToken = true
if avail, ok := reposFound["items_available"].(float64); !ok {
statusCode, statusText = http.StatusInternalServerError, "bad list response from API"
commit 82471a150c4d3a98f96c3a64f3725117f4229aee
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Dec 12 18:56:07 2017 -0500
8311: Add git_tree mount type.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 7d1d4c9..1e11aa2 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -323,6 +323,9 @@ start_api() {
}
start_nginx_proxy_services() {
+ if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
+ return
+ fi
echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
cd "$WORKSPACE" \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
@@ -904,7 +907,7 @@ if [ ! -z "$only" ] && [ "$only" == "services/api" ]; then
exit_cleanly
fi
-start_api || { stop_services; fatal "start_api"; }
+start_api && start_nginx_proxy_services || { stop_services; fatal "start_api"; }
test_ruby_sdk() {
cd "$WORKSPACE/sdk/ruby" \
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index a38d95c..24f3faa 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -245,6 +245,7 @@ type DiscoveryDocument struct {
BasePath string `json:"basePath"`
DefaultCollectionReplication int `json:"defaultCollectionReplication"`
BlobSignatureTTL int64 `json:"blobSignatureTtl"`
+ GitURL string `json:"gitUrl"`
Schemas map[string]Schema `json:"schemas"`
Resources map[string]Resource `json:"resources"`
}
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index 7e588be..16726b7 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -32,6 +32,7 @@ type Mount struct {
Content interface{} `json:"content"`
ExcludeFromOutput bool `json:"exclude_from_output"`
Capacity int64 `json:"capacity"`
+ Commit string `json:"commit"` // only if kind=="git_tree"
}
// RuntimeConstraints specify a container's compute resources (RAM,
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 7858fa0..fcb8b00 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -30,6 +30,11 @@ const (
Dispatch1AuthUUID = "zzzzz-gj3su-k9dvestay1plssr"
QueuedContainerUUID = "zzzzz-dz642-queuedcontainer"
+
+ ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
+ ArvadosRepoName = "arvados"
+ FooRepoUUID = "zzzzz-s0uqq-382brsig8rp3666"
+ FooRepoName = "active/foo"
)
// PathologicalManifest : A valid manifest designed to test
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index a237829..f413755 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -60,6 +60,7 @@ class Arvados::V1::SchemaController < ApplicationController
websocketUrl: Rails.application.config.websocket_address,
workbenchUrl: Rails.application.config.workbench_address,
keepWebServiceUrl: Rails.application.config.keep_web_service_url,
+ gitUrl: Rails.application.config.git_repo_https_base,
parameters: {
alt: {
type: "string",
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index f3f754b..3cddbad 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -538,6 +538,22 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
return fmt.Errorf("writing temp file: %v", err)
}
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", tmpfn, bind))
+
+ case mnt.Kind == "git_tree":
+ tmpdir, err := runner.MkTempDir("", "")
+ if err != nil {
+ return fmt.Errorf("creating temp dir: %v", err)
+ }
+ runner.CleanupTempDir = append(runner.CleanupTempDir, tmpdir)
+ err = GitMount(mnt).ExtractTree(runner.ArvClient, tmpdir)
+ if err != nil {
+ return err
+ }
+ bind := tmpdir + ":" + bind
+ if !mnt.Writable {
+ bind = bind + ":ro"
+ }
+ runner.Binds = append(runner.Binds, bind)
}
}
diff --git a/services/crunch-run/git_mount.go b/services/crunch-run/git_mount.go
new file mode 100644
index 0000000..5408382
--- /dev/null
+++ b/services/crunch-run/git_mount.go
@@ -0,0 +1,90 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "log"
+ "net/url"
+ "os"
+ "path/filepath"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ git "gopkg.in/src-d/go-git.v3"
+ git_http "gopkg.in/src-d/go-git.v3/clients/http"
+ "gopkg.in/src-d/go-git.v3/core"
+)
+
+type GitMount arvados.Mount
+
+// ExtractTree extracts the specified tree into dir, which is an
+// existing empty local directory.
+func (gm GitMount) ExtractTree(ac IArvadosClient, dir string) error {
+ if gm.Path != "/" {
+ return fmt.Errorf("cannot mount git_tree path %q -- only \"/\" is supported", gm.Path)
+ }
+ baseURL, err := ac.Discovery("gitUrl")
+ if err != nil {
+ return fmt.Errorf("error getting gitUrl from discovery doc: %s", err)
+ }
+ u, err := url.Parse(baseURL.(string))
+ if err != nil {
+ return fmt.Errorf("error parsing gitUrl %q: %s", baseURL, err)
+ }
+ u, err = u.Parse("/" + gm.UUID)
+ if err != nil {
+ return fmt.Errorf("error building git url from %q, %q: %s", baseURL, gm.UUID, err)
+ }
+ r, err := git.NewRepository(u.String(), git_http.NewBasicAuth("none", arvadostest.ActiveToken))
+ if err != nil {
+ return fmt.Errorf("error initializing repository: %s", err)
+ }
+ if err := r.PullDefault(); err != nil {
+ return fmt.Errorf("error pulling remote: %s", err)
+ }
+ c, err := r.Commit(core.NewHash(gm.Commit))
+ if err != nil {
+ return fmt.Errorf("error finding commit %q: %s", gm.Commit, err)
+ }
+ tw := git.NewTreeWalker(r, c.Tree())
+ defer tw.Close()
+ for {
+ name, entry, obj, err := tw.Next()
+ if err == io.EOF {
+ return nil
+ } else if err != nil {
+ return fmt.Errorf("Walker: %s", err)
+ }
+ switch o := obj.(type) {
+ case *git.Tree:
+ log.Printf("DEBUG: tree %q, mode %o", name, entry.Mode)
+ case *git.Blob:
+ err = writeBlobToFile(filepath.Join(dir, name), entry.Mode, o)
+ if err != nil {
+ return err
+ }
+ }
+ }
+}
+
+func writeBlobToFile(outName string, mode os.FileMode, blob *git.Blob) error {
+ r, err := blob.Reader()
+ if err != nil {
+ return fmt.Errorf("error reading blob %q: %s", blob.ID(), err)
+ }
+ defer r.Close()
+ f, err := os.OpenFile(outName, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
+ if err != nil {
+ return fmt.Errorf("error creating local file %q: %s", outName, err)
+ }
+ n, err := io.Copy(f, r)
+ if err != nil {
+ f.Close()
+ return fmt.Errorf("error copying blob %q to local file %q after %d bytes: %s", blob.ID(), outName, n, err)
+ }
+ err = f.Close()
+ if err != nil {
+ return fmt.Errorf("error closing local file %q: %s", outName, err)
+ }
+ return nil
+}
diff --git a/services/crunch-run/git_mount_test.go b/services/crunch-run/git_mount_test.go
new file mode 100644
index 0000000..5aa4cd6
--- /dev/null
+++ b/services/crunch-run/git_mount_test.go
@@ -0,0 +1,59 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "io/ioutil"
+ "net/http"
+ "os"
+ "path/filepath"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ check "gopkg.in/check.v1"
+ git_clients "gopkg.in/src-d/go-git.v3/clients"
+ git_http "gopkg.in/src-d/go-git.v3/clients/http"
+)
+
+type GitMountSuite struct{}
+
+var _ = check.Suite(&GitMountSuite{})
+
+func (s *GitMountSuite) TestExtract(c *check.C) {
+ origClient := http.DefaultClient
+ defer func() { http.DefaultClient = origClient }()
+ http.DefaultClient = arvados.InsecureHTTPClient
+
+ git_clients.KnownProtocols["https"] = &git_http.GitUploadPackService{
+ Client: arvados.InsecureHTTPClient,
+ }
+
+ tmpdir, err := ioutil.TempDir("", "")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tmpdir)
+
+ ac := &ArvTestClient{}
+ port, err := ioutil.ReadFile("../../tmp/arv-git-httpd-ssl.port")
+ c.Assert(err, check.IsNil)
+ discoveryMap["gitUrl"] = "https://localhost:" + string(port)
+ gm := GitMount{
+ Path: "/",
+ UUID: arvadostest.FooRepoUUID,
+ Commit: "1de84a854e2b440dc53bf42f8548afa4c17da332",
+ }
+ err = gm.ExtractTree(ac, tmpdir)
+ c.Check(err, check.IsNil)
+
+ data, err := ioutil.ReadFile(filepath.Join(tmpdir, "foo"))
+ c.Check(err, check.IsNil)
+ c.Check(string(data), check.Equals, "branch-foo3\n")
+
+ c.Check(false, check.Equals, "TODO: test file mode other than 0644")
+ c.Check(false, check.Equals, "TODO: test file inside a directory")
+ c.Check(false, check.Equals, "TODO: test file inside a folder")
+ c.Check(false, check.Equals, "TODO: test repo not found")
+ c.Check(false, check.Equals, "TODO: test commit does not exist")
+ c.Check(false, check.Equals, "TODO: test commit not on master branch")
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list