[ARVADOS] updated: b4f4cbf7ef41492d36c81eb83239f6e4c6744a6a

git at public.curoverse.com git at public.curoverse.com
Tue Mar 17 11:35:04 EDT 2015


Summary of changes:
 apps/workbench/Gemfile                             |  2 +-
 apps/workbench/Gemfile.lock                        |  4 +-
 .../app/assets/javascripts/application.js          |  3 +
 .../app/assets/stylesheets/application.css.scss    | 10 +++
 .../application/_show_autoselect_text.html.erb     |  4 ++
 .../collections/_show_source_summary.html.erb      |  4 ++
 apps/workbench/app/views/collections/show.html.erb | 23 +++----
 .../views/projects/_compute_node_status.html.erb   |  4 +-
 .../test/integration/collection_upload_test.rb     |  3 +-
 .../workbench/test/integration/collections_test.rb | 11 ++--
 apps/workbench/test/integration/projects_test.rb   |  1 -
 doc/_config.yml                                    |  7 +--
 doc/_includes/_navbar_top.liquid                   |  7 +--
 doc/admin/index.html.md.liquid                     | 14 -----
 doc/index.html.liquid                              |  3 -
 .../cheat_sheet.html.textile.liquid                | 10 +--
 doc/install/install-api-server.html.textile.liquid |  2 +-
 .../install-arv-git-httpd.html.textile.liquid      | 73 ++++++++++++++++++++++
 .../install-workbench-app.html.textile.liquid      |  2 +-
 sdk/cli/bin/crunch-job                             |  9 +--
 sdk/ruby/lib/arvados/keep.rb                       |  5 ++
 sdk/ruby/test/test_keep_manifest.rb                | 13 ++++
 .../controllers/arvados/v1/schema_controller.rb    |  1 +
 services/api/config/application.default.yml        |  8 +++
 ...tion_portable_data_hash_with_hinted_manifest.rb |  9 ++-
 ...136_change_collection_expires_at_to_datetime.rb |  9 +++
 services/api/db/structure.sql                      |  6 +-
 services/arv-git-httpd/server_test.go              | 24 +++++--
 services/keepstore/keepstore.go                    | 16 ++---
 29 files changed, 208 insertions(+), 79 deletions(-)
 create mode 100644 apps/workbench/app/views/application/_show_autoselect_text.html.erb
 delete mode 100644 doc/admin/index.html.md.liquid
 rename doc/{admin => install}/cheat_sheet.html.textile.liquid (70%)
 create mode 100644 doc/install/install-arv-git-httpd.html.textile.liquid
 create mode 100644 services/api/db/migrate/20150312151136_change_collection_expires_at_to_datetime.rb

       via  b4f4cbf7ef41492d36c81eb83239f6e4c6744a6a (commit)
       via  ad69cab04c31db7ed78ced64e3e1fa814775d6d7 (commit)
       via  7521d023fbeb1cabc9e89953e29cd55183af3753 (commit)
       via  ae8ed6d9d0e5231379564beedc08c812352b7043 (commit)
       via  e3f359d1cf36bca2b675c495fcff0a76a89cbd38 (commit)
       via  382924ca272e477a3739aec567fa2177e33da6fa (commit)
       via  9877c8914fa0bb17fcb9d6f2e30e067f8b135d79 (commit)
       via  4f3b7339bef1286a34745c7ecd97476f56af469c (commit)
       via  c3074f48c15ae7d1f4bc30939959c7243708cb37 (commit)
       via  1321195ae33f9aafc5b9c328b9352b6b5277de37 (commit)
       via  d16e54da7e751807685f576d089d69417c9094b0 (commit)
       via  b306eb48ab12676ffb365ede8197e4f2d7e92011 (commit)
       via  fcd19228447a2983c7aba8a0f09984eba06ded48 (commit)
       via  a9b4a906124081439326fedc1a1e73fae9c2f40b (commit)
       via  9f112e10e7287bbad4e199d5092965f4b0d1cde2 (commit)
       via  dc94a5b530afe42b7060d0188d04423a7cf7a205 (commit)
       via  f1bb6330f94f341c0d53eb01da532bfa995924f4 (commit)
       via  1e54d655b91afae99816f47ac3b9283ae12ba4c2 (commit)
       via  71c4fdc3352ad5ca34c2f260fe43fa1150868a04 (commit)
       via  2f27d8fa4c8f10989d1066df4df9c8995ee755f5 (commit)
       via  1bda9759f558316f01894634504a4346085be24d (commit)
       via  a7f932428b30582f54ea26084813eb7b3e8f3251 (commit)
       via  6e066d1a18a699e7abb20dda25670f814e1eb3b3 (commit)
       via  e893c4e1c4cb3b5f564517d281906f28e8799617 (commit)
       via  40d408b31405c780160490ddcc104be7d5a99368 (commit)
       via  a0f20c7a3e15b30efb1c48671a0fd4e1d87b21bc (commit)
       via  995af457bf39bfa82556a6f565efd30263a98415 (commit)
       via  f3e7d40574c721225c280de14394ffd5b97ab4ab (commit)
       via  09970b422d59e8fbe84e17a1146279d1e090e670 (commit)
       via  9a38a7c6881e36523d2e056cab1781fcfec4af8e (commit)
       via  bf213b6340f66bec945ceab8d2bab59f7ca17a6f (commit)
      from  b93018f033f37a7a7b719fd7dc6d5b3ad4a0f797 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit b4f4cbf7ef41492d36c81eb83239f6e4c6744a6a
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue Mar 17 11:36:07 2015 -0400

    5416: Add arv-git-httpd to install docs, api config, and discovery doc.

diff --git a/doc/_config.yml b/doc/_config.yml
index c1b7983..40ac71b 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -149,6 +149,7 @@ navbar:
       - install/create-standard-objects.html.textile.liquid
       - install/install-keepstore.html.textile.liquid
       - install/install-keepproxy.html.textile.liquid
+      - install/install-arv-git-httpd.html.textile.liquid
       - install/install-crunch-dispatch.html.textile.liquid
       - install/cheat_sheet.html.textile.liquid
     - Software prerequisites:
diff --git a/doc/install/install-arv-git-httpd.html.textile.liquid b/doc/install/install-arv-git-httpd.html.textile.liquid
new file mode 100644
index 0000000..85f643f
--- /dev/null
+++ b/doc/install/install-arv-git-httpd.html.textile.liquid
@@ -0,0 +1,73 @@
+---
+layout: default
+navsection: installguide
+title: Install Git server
+...
+
+This installation guide assumes you are on a 64 bit Debian or Ubuntu system.
+
+The arv-git-httpd server provides HTTP access to hosted git repositories, using Arvados authentication tokens instead of passwords. It is intended to be installed on the system where your git repositories are stored, and accessed through a web proxy that provides SSL support.
+
+By convention, we use the following hostname for the git service:
+
+<div class="offset1">
+table(table table-bordered table-condensed).
+|git. at uuid_prefix@.your.domain|
+</div>
+
+This hostname should resolve from anywhere on the internet.
+
+h2. Install arv-git-httpd
+
+First add the Arvados apt repository, and then install the arv-git-httpd package.
+
+<notextile>
+<pre><code>~$ <span class="userinput">echo "deb http://apt.arvados.org/ wheezy main" | sudo tee /etc/apt/sources.list.d/apt.arvados.org.list</span>
+~$ <span class="userinput">sudo /usr/bin/apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7</span>
+~$ <span class="userinput">sudo /usr/bin/apt-get update</span>
+~$ <span class="userinput">sudo /usr/bin/apt-get install arv-git-httpd</span>
+</code></pre>
+</notextile>
+
+Verify that @arv-git-httpd@ and @git-http-backend@ are functional:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv-git-httpd -h</span>
+Usage of arv-git-httpd:
+  -address="0.0.0.0:80": Address to listen on, "host:port".
+  -git-command="/usr/bin/git": Path to git executable. Each authenticated request will execute this program with a single argument, "http-backend".
+  -repo-root="/path/to/cwd": Path to git repositories.
+~$ <span class="userinput">git http-backend</span>
+Status: 500 Internal Server Error
+Expires: Fri, 01 Jan 1980 00:00:00 GMT
+Pragma: no-cache
+Cache-Control: no-cache, max-age=0, must-revalidate
+
+fatal: No REQUEST_METHOD from server
+</code></pre>
+</notextile>
+
+We recommend running @arv-git-httpd@ under "runit":https://packages.debian.org/search?keywords=runit or something similar.
+
+Your @run@ script should look something like this:
+
+<notextile>
+<pre><code>export ARVADOS_API_HOST=<span class="userinput">uuid_prefix</span>.your.domain
+exec sudo -u git arv-git-httpd -address=:9001 -git-command="$(which git)" -repo-root=<span class="userinput">/var/lib/arvados/git</span> 2>&1
+</code></pre>
+</notextile>
+
+h3. Set up a reverse proxy with SSL support
+
+The arv-git-httpd service will be accessible from anywhere on the internet, so we recommend using SSL for transport encryption.
+
+This is best achieved by putting a reverse proxy with SSL support in front of arv-git-httpd, running on port 443 and passing requests to arv-git-httpd on port 9001 (or whatever port you chose in your run script).
+
+h3. Tell the API server about the arv-git-httpd service
+
+In your API server's config/application.yml file, add the following entry:
+
+<notextile>
+<pre><code>git_http_base: git.<span class="userinput">uuid_prefix</span>.your.domain
+</code></pre>
+</notextile>
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 8dc90e1..dbdc382 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -26,6 +26,7 @@ class Arvados::V1::SchemaController < ApplicationController
         description: "The API to interact with Arvados.",
         documentationLink: "http://doc.arvados.org/api/index.html",
         defaultCollectionReplication: Rails.configuration.default_collection_replication,
+        gitHttpBase: Rails.configuration.git_http_base,
         protocol: "rest",
         baseUrl: root_url + "arvados/v1/",
         basePath: "/arvados/v1/",
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 8ecef11..b6d37f9 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -69,6 +69,11 @@ common:
   # {git_repositories_dir}/arvados/.git
   git_repositories_dir: /var/lib/arvados/git
 
+  # If an arv-git-httpd service is running, advertise it in the
+  # discovery document by adding its public URI base here. Example:
+  # https://git.xxxxx.arvadosapi.com
+  git_http_base: false
+
   # This is a (bare) repository that stores commits used in jobs.  When a job
   # runs, the source commits are first fetched into this repository, then this
   # repository is used to deploy to compute nodes.  This should NOT be a

commit ad69cab04c31db7ed78ced64e3e1fa814775d6d7
Merge: 7521d02 ae8ed6d
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue Mar 17 10:57:35 2015 -0400

    5416: Merge branch 'master' into 5416-git-auth-token


commit 7521d023fbeb1cabc9e89953e29cd55183af3753
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Mar 16 17:37:45 2015 -0400

    5416: Add path variants test. Skip cloning arvados repo when not needed.

diff --git a/services/arv-git-httpd/server_test.go b/services/arv-git-httpd/server_test.go
index c3eed3d..9d2c190 100644
--- a/services/arv-git-httpd/server_test.go
+++ b/services/arv-git-httpd/server_test.go
@@ -21,6 +21,16 @@ type IntegrationSuite struct {
 	testServer  *server
 }
 
+func (s *IntegrationSuite) TestPathVariants(c *check.C) {
+	s.makeArvadosRepo(c)
+	// Spectator token
+	os.Setenv("ARVADOS_API_TOKEN", "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu")
+	for _, repo := range []string{"foo.git", "foo/.git", "arvados.git", "arvados/.git"} {
+		err := s.runGit(c, "fetch", repo)
+		c.Assert(err, check.Equals, nil)
+	}
+}
+
 func (s *IntegrationSuite) TestReadonly(c *check.C) {
 	// Spectator token
 	os.Setenv("ARVADOS_API_TOKEN", "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu")
@@ -53,7 +63,7 @@ func (s *IntegrationSuite) TestNonexistent(c *check.C) {
 func (s *IntegrationSuite) TestNoPermission(c *check.C) {
 	// Anonymous token
 	os.Setenv("ARVADOS_API_TOKEN", "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi")
-	for _, repo := range []string{"foo.git", "foo/.git", "foo/bar.git", "foo/bar/.git"} {
+	for _, repo := range []string{"foo.git", "foo/.git"} {
 		err := s.runGit(c, "fetch", repo)
 		c.Assert(err, check.ErrorMatches, `.* not found:.*`)
 	}
@@ -71,10 +81,6 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
 	c.Assert(err, check.Equals, nil)
 	s.tmpWorkdir, err = ioutil.TempDir("", "arv-git-httpd")
 	c.Assert(err, check.Equals, nil)
-	_, err = exec.Command("git", "init", "--bare", s.tmpRepoRoot + "/arvados.git").Output()
-	c.Assert(err, check.Equals, nil)
-	_, err = exec.Command("git", "--git-dir", s.tmpRepoRoot + "/arvados.git", "fetch", "../../.git", "master:master").Output()
-	c.Assert(err, check.Equals, nil)
 	_, err = exec.Command("git", "init", s.tmpRepoRoot + "/foo").Output()
 	c.Assert(err, check.Equals, nil)
 	_, err = exec.Command("sh", "-c", "cd " + s.tmpRepoRoot + "/foo && echo test >test && git add test && git commit -am 'foo: test'").CombinedOutput()
@@ -146,6 +152,14 @@ func (s *IntegrationSuite) runGit(c *check.C, gitCmd, repo string, args ...strin
 	return err
 }
 
+// Make a bare arvados repo at {tmpRepoRoot}/arvados.git
+func (s *IntegrationSuite) makeArvadosRepo(c *check.C) {
+	_, err := exec.Command("git", "init", "--bare", s.tmpRepoRoot + "/arvados.git").Output()
+	c.Assert(err, check.Equals, nil)
+	_, err = exec.Command("git", "--git-dir", s.tmpRepoRoot + "/arvados.git", "fetch", "../../.git", "master:master").Output()
+	c.Assert(err, check.Equals, nil)
+}
+
 // Gocheck boilerplate
 func Test(t *testing.T) {
 	check.TestingT(t)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list