[ARVADOS] updated: 8a4463cb39fae171d042c626485efce4ff05917d
git at public.curoverse.com
git at public.curoverse.com
Thu Jul 30 16:18:04 EDT 2015
Summary of changes:
apps/workbench/Gemfile.lock | 3 -
.../app/helpers/pipeline_instances_helper.rb | 4 +-
.../pipeline_instances/_running_component.html.erb | 16 +--
.../_show_components_running.html.erb | 24 ++--
.../test/integration/pipeline_instances_test.rb | 2 +-
doc/_includes/_arv_copy_expectations.liquid | 5 +-
.../create-standard-objects.html.textile.liquid | 4 +-
.../install-arv-git-httpd.html.textile.liquid | 35 +++---
.../install-shell-server.html.textile.liquid | 10 +-
sdk/cli/bin/crunch-job | 26 +++--
sdk/cli/test/binstub_clean_fail/mount | 3 +
sdk/cli/test/binstub_docker_noop/docker.io | 2 +
sdk/cli/test/binstub_sanity_check/docker.io | 2 +
sdk/cli/test/binstub_sanity_check/true | 2 +
sdk/cli/test/test_arv-collection-create.rb | 2 -
sdk/cli/test/test_arv-get.rb | 42 +------
sdk/cli/test/test_arv-put.rb | 26 -----
sdk/cli/test/test_arv-run-pipeline-instance.rb | 2 -
sdk/cli/test/test_arv-tag.rb | 2 +-
sdk/cli/test/test_crunch-job.rb | 126 +++++++++++++++++++++
sdk/perl/Makefile.PL | 7 +-
sdk/python/arvados/commands/arv_copy.py | 11 ++
.../arvados/v1/repositories_controller.rb | 92 ++++++++++-----
23 files changed, 281 insertions(+), 167 deletions(-)
create mode 100755 sdk/cli/test/binstub_clean_fail/mount
create mode 100755 sdk/cli/test/binstub_docker_noop/docker.io
create mode 100755 sdk/cli/test/binstub_sanity_check/docker.io
create mode 100755 sdk/cli/test/binstub_sanity_check/true
create mode 100644 sdk/cli/test/test_crunch-job.rb
discards a53880a2f3f24fd58d7604f575a0a8b83908fe4c (commit)
via 8a4463cb39fae171d042c626485efce4ff05917d (commit)
via 6dff0705fd3b4e0acde7bdf5821ef115ba74099b (commit)
via 39fe0413aa14ff0ba41d790441c8621a877f61f8 (commit)
via 52c8fd376a75558657969cdf11955334e95d9e83 (commit)
via 8797ab137a1d8824292a6f7202f6cd0c9744c4f9 (commit)
via 8699ba0baf2e346f0adc7ca79f0a8b129f56b9a7 (commit)
via 747e22c9341d00a3f093298a979175166e2898a7 (commit)
via a3d4fc92c1cfc2431adb55255793e0077c0a5bd1 (commit)
via eba475a918e4ae847ae6b5753016f2eb54e18085 (commit)
via ec752f92eff3990718d169fccc06a04ddc089578 (commit)
via db66ff7c2511f80f3795c388f314753064378543 (commit)
via ccf30f40f46ea450d7ab3766f0923b486a7450d3 (commit)
via 21f8e36579eb7091041cf696d1e7fbb14caeb6ea (commit)
via 7d5d40c55d2a38b12e810f3b9d3e168ee434cbd2 (commit)
via 5058d1cbdbde801a2cf7e303d83e1a626015afdd (commit)
via 06d6b29b121b80cb975e5d9d29a9a6f9b3763c47 (commit)
via 3be96c96a4ff1c519e03e6dc2036e0b93e41ee8c (commit)
via 2b7d9f3f2e687d524ccc1d88e470e721f63f9e51 (commit)
via ec64034683c31d76d02c293f0ae27d6ae80653cb (commit)
via 23d0c610d06eaabca9efa0ce36a672973b3e84c9 (commit)
via 0c447acb37a221a096249c6a9cfad4255c08f575 (commit)
via e39e11d5850c1a2509f377e2f2551105507f59e3 (commit)
via 56d94b28fc1ee2fe2f36f6743186e8b442c9e676 (commit)
via 3282d76febc482aab302a44bf594bcf9591ff868 (commit)
via fd1c8009f521564eaec4dbb22cead032e3784023 (commit)
via 6d2811a2e001369b950c452be3dee14db8d795c2 (commit)
via c86a3350fe212e35a7d745d5607fbc27bc1fd3c8 (commit)
via aa924bec2e4b5c029cb19f4e9021d72b665be76b (commit)
via 9ce68afa9f3f1b558acd19561760c19e35b8a356 (commit)
via 1b9bb00665f837cbafc87bb6fca2252e453abefc (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (a53880a2f3f24fd58d7604f575a0a8b83908fe4c)
\
N -- N -- N (8a4463cb39fae171d042c626485efce4ff05917d)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8a4463cb39fae171d042c626485efce4ff05917d
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Jul 26 13:08:55 2015 -0400
6663: Document git setup.
diff --git a/doc/install/create-standard-objects.html.textile.liquid b/doc/install/create-standard-objects.html.textile.liquid
index 1ce4fae..77cc4ba 100644
--- a/doc/install/create-standard-objects.html.textile.liquid
+++ b/doc/install/create-standard-objects.html.textile.liquid
@@ -5,7 +5,7 @@ title: Create standard objects
...
-Next, we're going to use the Arvados CLI tools on the <strong>shell server</strong> to create a few Arvados objects.
+In these steps we use the Arvados CLI tools on the <strong>shell server</strong> to create a few Arvados objects. The CLI tools require an ARVADOS_API_TOKEN environment variable with a valid admin token. If you haven't already done so, set that up as shown in the "API token guide":../user/reference/api-tokens.html.
First we create a repository object which will be used to set up a hosted clone of the arvados repository on this cluster.
@@ -30,6 +30,15 @@ Secondly, create a link object to make the repository object readable by the "Al
EOF</span>
</code></pre></notextile>
+In a couple of minutes, your arvados-git-sync cron job will create an empty repository on your git server. Seed it with the real arvados repository. If your git credential helpers were configured correctly when you "set up your shell server":install-shell-server.html, the "git push" command will use your API token instead of prompting you for a username and password.
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd /tmp</span>
+/tmp$ <span class="userinput">git clone --bare https://github.com/curoverse/arvados.git</span>
+/tmp/arvados$ <span class="userinput">git --git-dir arvados.git push https://git.<b>uuid_prefix.your.domain</b>/arvados.git '*:*'</span>
+</code></pre>
+</notextile>
+
Next, create a default project for the standard Arvados Docker images, and give all users read access to it. The project is owned by the system user.
<notextile>
diff --git a/doc/install/install-api-server.html.textile.liquid b/doc/install/install-api-server.html.textile.liquid
index 0503609..f5d5825 100644
--- a/doc/install/install-api-server.html.textile.liquid
+++ b/doc/install/install-api-server.html.textile.liquid
@@ -94,13 +94,10 @@ Define your @uuid_prefix@ in @application.yml@ by setting the @uuid_prefix@ fiel
h3(#git_repositories_dir). git_repositories_dir
-This field defaults to @/var/lib/arvados/git at . You can override the value by defining it in @application.yml at .
-
-Make sure a clone of the arvados repository exists in @git_repositories_dir at .
+Additional git setup will be done on the "next page":install-arv-git-httpd.html. For now, just create the repository storage directory.
<notextile>
-<pre><code>~$ <span class="userinput">sudo mkdir -p /var/lib/arvados/git</span>
-~$ <span class="userinput">sudo git clone --bare git://git.curoverse.com/arvados.git /var/lib/arvados/git/arvados.git</span>
+<pre><code>~$ <span class="userinput">sudo mkdir -p /var/lib/arvados/git/repositories</span>
</code></pre></notextile>
h3. secret_token
@@ -146,9 +143,10 @@ h2. Prepare the API server deployment
Now that all your configuration is in place, run @/usr/local/bin/arvados-api-server-upgrade.sh at . This will install and check your configuration, install necessary gems, and run any necessary database setup.
{% include 'notebox_begin' %}
-You can safely ignore the following error message you may see when loading the database structure:
-<notextile>
-<pre><code>ERROR: must be owner of extension plpgsql</code></pre></notextile>
+You can safely ignore the following messages if they appear while this script runs:
+<pre>Don't run Bundler as root. Bundler can ask for sudo if it is needed, and installing your bundle as root will
+break this application for all non-root users on this machine.</pre>
+<pre>fatal: Not a git repository (or any of the parent directories): .git</pre>
{% include 'notebox_end' %}
This command aborts when it encounters an error. It's safe to rerun multiple times, so if there's a problem with your configuration, you can fix that and try again.
diff --git a/doc/install/install-arv-git-httpd.html.textile.liquid b/doc/install/install-arv-git-httpd.html.textile.liquid
index 33b1124..b8945f9 100644
--- a/doc/install/install-arv-git-httpd.html.textile.liquid
+++ b/doc/install/install-arv-git-httpd.html.textile.liquid
@@ -1,21 +1,220 @@
---
layout: default
navsection: installguide
-title: Install Git server
+title: Install the Git server
...
-The arvados-git-httpd server provides HTTP access to hosted git repositories, using Arvados authentication tokens instead of passwords. It is intended to be installed on the system where your git repositories are stored, and accessed through a web proxy that provides SSL support.
+Arvados allows users to create their own private and public git repositories, and clone/push them using SSH and HTTPS.
+
+The git hosting setup involves three components.
+* The "arvados-git-sync.rb" script polls the API server for the current list of repositories, creates bare repositories, and updates the local permission cache used by gitolite.
+* Gitolite provides SSH access.
+* arvados-git-http provides HTTPS access.
+
+It is not strictly necessary to deploy _both_ SSH and HTTPS access, but we recommend deploying both:
+* SSH is a more appropriate way to authenticate from a user's workstation because it does not require managing tokens on the client side;
+* HTTPS is a more appropriate way to authenticate from a shell VM because it does not depend on SSH agent forwarding (SSH clients' agent forwarding features tend to behave as if the remote machine is fully trusted).
+
+The HTTPS instructions given below will not work if you skip the SSH setup steps.
+
+h2. Set up DNS
By convention, we use the following hostname for the git service:
-<div class="offset1">
-table(table table-bordered table-condensed).
-|git. at uuid_prefix@.your.domain|
-</div>
+<notextile>
+<pre><code>git.<span class="userinput">uuid_prefix</span>.your.domain
+</code></pre>
+</notextile>
+
+{% include 'notebox_begin' %}
+Here, we show how to install the git hosting services *on the same host as your API server.* Using a different host is not yet fully supported. On this page we will refer to it as your git server.
+{% include 'notebox_end' %}
+
+DNS and network configuration should be set up so port 443 reaches your HTTPS proxy, and port 22 reaches your git server.
+
+h2. Generate an API token
+
+On the API server, if you are using RVM:
+
+<notextile>
+<pre><code>gitserver:~$ <span class="userinput">cd /var/www/arvados-api/current</span>
+gitserver:/var/www/arvados-api/current$ <span class="userinput">sudo -u www-data RAILS_ENV=production `which rvm-exec` default bundle exec ./script/create_superuser_token.rb</span>
+4hdqaixi5a027jqn0vyjbwa3xmcue8logzhtsmk1bplgp064fe
+</code></pre>
+</notextile>
+
+If you are not using RVM:
+
+<notextile>
+<pre><code>gitserver:~$ <span class="userinput">cd /var/www/arvados-api/current</span>
+gitserver:/var/www/arvados-api/current$ <span class="userinput">sudo -u www-data RAILS_ENV=production bundle exec ./script/create_superuser_token.rb</span>
+4hdqaixi5a027jqn0vyjbwa3xmcue8logzhtsmk1bplgp064fe
+</code></pre>
+</notextile>
+
+Copy that token; you'll need it in a minute.
+
+h2. Install git and other dependencies
+
+On Debian-based systems:
+
+<notextile>
+<pre><code>gitserver:~$ <span class="userinput">sudo apt-get install git openssh-server</span>
+</code></pre>
+</notextile>
+
+On Red Hat-based systems:
+
+<notextile>
+<pre><code>gitserver:~$ <span class="userinput">sudo yum install git perl-Data-Dumper openssh-server</span>
+</code></pre>
+</notextile>
+
+h2. Create a "git" user and a storage directory
+
+Hosted repositories will be stored under @/var/lib/arvados/git/@. If you choose a different location, make sure to update the @git_repositories_dir@ entry in your API server's @config/application.yml@ file, preserving the trailing @repositories/@ part.
+
+A new UNIX account called "git" will own the files. This makes git URLs look familiar to users (<code>git@[...]:username/reponame.git</code>).
+
+On Debian- or Red Hat-based systems:
+
+<notextile>
+<pre><code>gitserver:~$ <span class="userinput">sudo mkdir -p /var/lib/arvados/git/repositories</span>
+gitserver:~$ <span class="userinput">sudo useradd --comment git --home-dir /var/lib/arvados/git git</span>
+gitserver:~$ <span class="userinput">sudo chown -R git:git ~git</span>
+</code></pre>
+</notextile>
+
+The git user needs its own SSH key. (It must be able to run @ssh git at localhost@ from scripts.)
+
+<notextile>
+<pre><code>gitserver:~$ <span class="userinput">sudo -u git -i bash</span>
+git at gitserver:~$ <span class="userinput">ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa</span>
+git at gitserver:~$ <span class="userinput">ssh -o stricthostkeychecking=no localhost cat .ssh/id_rsa.pub</span>
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7aBIDAAgMQN16Pg6eHmvc+D+6TljwCGr4YGUBphSdVb25UyBCeAEgzqRiqy0IjQR2BLtSirXr+1SJAcQfBgI/jwR7FG+YIzJ4ND9JFEfcpq20FvWnMMQ6XD3y3xrZ1/h/RdBNwy4QCqjiXuxDpDB7VNP9/oeAzoATPZGhqjPfNS+RRVEQpC6BzZdsR+S838E53URguBOf9yrPwdHvosZn7VC0akeWQerHqaBIpSfDMtaM4+9s1Gdsz0iP85rtj/6U/K/XOuv2CZsuVZZ52nu3soHnEX2nx2IaXMS3L8Z+lfOXB2T6EaJgXF7Z9ME5K1tx9TSNTRcYCiKztXLNLSbp git at gitserver
+git at gitserver:~$ <span class="userinput">exit</span>
+</code></pre>
+</notextile>
+
+h2. Install gitolite
+
+Check https://github.com/sitaramc/gitolite/tags for the latest stable version (_e.g.,_ @v3.6.3@).
-This hostname should resolve from anywhere on the internet.
+Download and install the version you selected.
-h2. Install arvados-git-httpd
+<notextile>
+<pre><code>gitserver:~$ <span class="userinput">sudo -u git -i bash</span>
+git at gitserver:~$ <span class="userinput">echo 'PATH=$HOME/bin:$PATH' >.profile</span>
+git at gitserver:~$ <span class="userinput">source .profile</span>
+git at gitserver:~$ <span class="userinput">git clone --branch <b>v3.6.3</b> git://github.com/sitaramc/gitolite</span>
+...
+Note: checking out '5d24ae666bfd2fa9093d67c840eb8d686992083f'.
+...
+git at gitserver:~$ <span class="userinput">mkdir bin</span>
+git at gitserver:~$ <span class="userinput">gitolite/install -ln ~git/bin</span>
+git at gitserver:~$ <span class="userinput">bin/gitolite setup -pk .ssh/id_rsa.pub</span>
+Initialized empty Git repository in /var/lib/arvados/git/repositories/gitolite-admin.git/
+Initialized empty Git repository in /var/lib/arvados/git/repositories/testing.git/
+WARNING: /var/lib/arvados/git/.ssh missing; creating a new one
+ (this is normal on a brand new install)
+WARNING: /var/lib/arvados/git/.ssh/authorized_keys missing; creating a new one
+ (this is normal on a brand new install)
+</code></pre>
+</notextile>
+
+Create the base configuration files.
+
+<notextile>
+<pre><code>git at gitserver:~$ <span class="userinput">git clone git at localhost:gitolite-admin</span>
+Cloning into 'gitolite-admin'...
+warning: You appear to have cloned an empty repository.
+done.
+git at gitserver:~$ <span class="userinput">cd gitolite-admin</span>
+git at gitserver:~/gitolite-admin$ <span class="userinput">git config user.email arvados</span>
+git at gitserver:~/gitolite-admin$ <span class="userinput">git config user.name arvados</span>
+git at gitserver:~/gitolite-admin$ <span class="userinput">git config push.default simple</span>
+git at gitserver:~/gitolite-admin$ <span class="userinput">touch conf/gitolite.conf</span>
+git at gitserver:~/gitolite-admin$ <span class="userinput">git add .</span>
+git at gitserver:~/gitolite-admin$ <span class="userinput">git commit -m -</span>
+git at gitserver:~/gitolite-admin$ <span class="userinput">git push</span>
+[master a4d083b] -
+ 1 files changed, 0 insertions(+), 0 deletions(-)
+ create mode 100644 conf/gitolite.conf
+git at gitserver:~/gitolite-admin$ <span class="userinput">exit</span>
+</code></pre>
+</notextile>
+
+h2. Configure gitolite
+
+Configure gitolite to look up a repository name like @username/reponame.git@ and find the appropriate bare repository storage directory.
+
+Add the following lines to the top of @~git/.gitolite.rc@:
+
+<notextile>
+<pre><code><span class="userinput">my $repo_aliases;
+my $aliases_src = "$ENV{HOME}/.gitolite/arvadosaliases.pl";
+if ($ENV{HOME} && (-e $aliases_src)) {
+ $repo_aliases = do $aliases_src;
+}
+$repo_aliases ||= {};
+</span></code></pre>
+</notextile>
+
+Add the following lines inside the section that begins @%RC = (@:
+
+<notextile>
+<pre><code><span class="userinput"> REPO_ALIASES => $repo_aliases,
+</span></code></pre>
+</notextile>
+
+Uncomment the 'Alias' line in the section that begins @ENABLE => [@:
+
+<notextile>
+<pre><code><span class="userinput"> # access a repo by another (possibly legacy) name
+ 'Alias',
+</span></code></pre>
+</notextile>
+
+h2. Configure git synchronization
+
+Create a configuration file @/var/www/arvados-api/current/config/arvados-clients.yml@ using the following template, filling in the appropriate values for your system.
+* For @arvados_api_token@, use the token you generated above.
+* For @gitolite_arvados_git_user_key@, provide the public key you generated above, i.e., the contents of @~git/.ssh/id_rsa.pub at .
+
+<notextile>
+<pre><code>production:
+ gitolite_url: /var/lib/arvados/git/repositories/gitolite-admin.git
+ gitolite_tmp: /var/lib/arvados/git
+ arvados_api_host: <span class="userinput">uuid_prefix.example.com</span>
+ arvados_api_token: "<span class="userinput">4hdqaixi5a027jqn0vyjbwa3xmcue8logzhtsmk1bplgp064fe</span>"
+ arvados_api_host_insecure: <span class="userinput">false</span>
+ gitolite_arvados_git_user_key: "<span class="userinput">ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7aBIDAAgMQN16Pg6eHmvc+D+6TljwCGr4YGUBphSdVb25UyBCeAEgzqRiqy0IjQR2BLtSirXr+1SJAcQfBgI/jwR7FG+YIzJ4ND9JFEfcpq20FvWnMMQ6XD3y3xrZ1/h/RdBNwy4QCqjiXuxDpDB7VNP9/oeAzoATPZGhqjPfNS+RRVEQpC6BzZdsR+S838E53URguBOf9yrPwdHvosZn7VC0akeWQerHqaBIpSfDMtaM4+9s1Gdsz0iP85rtj/6U/K/XOuv2CZsuVZZ52nu3soHnEX2nx2IaXMS3L8Z+lfOXB2T6EaJgXF7Z9ME5K1tx9TSNTRcYCiKztXLNLSbp git at gitserver</span>"
+</code></pre>
+</notextile>
+
+h2. Enable the synchronization script
+
+The API server package includes a script that retrieves the current set of repository names and permissions from the API, writes names and permissions to @arvadosaliases.pl@ in a format usable by gitolite, and creates new empty repositories if needed. This script should run every 2 to 5 minutes.
+
+If you are using RVM, create @/etc/cron.d/arvados-git-sync@ with the following content:
+
+<notextile>
+<pre><code><span class="userinput">*/5 * * * * git cd /var/www/arvados-api/current && /usr/local/rvm/bin/rvm-exec default bundle exec script/arvados-git-sync.rb production</span>
+</code></pre>
+</notextile>
+
+Otherwise, create @/etc/cron.d/arvados-git-sync@ with the following content:
+
+<notextile>
+<pre><code><span class="userinput">*/5 * * * * git cd /var/www/arvados-api/current && bundle exec script/arvados-git-sync.rb production</span>
+</code></pre>
+</notextile>
+
+h2. Install the arvados-git-httpd package
+
+This is needed only for HTTPS access.
+
+The arvados-git-httpd package provides HTTP access, using Arvados authentication tokens instead of passwords. It is intended to be installed on the system where your git repositories are stored, and accessed through a web proxy that provides SSL support.
On Debian-based systems:
@@ -31,11 +230,11 @@ On Red Hat-based systems:
</code></pre>
</notextile>
-Verify that @arvados-git-httpd@ and @git-http-backend@ are functional:
+Verify that @arvados-git-httpd@ and @git-http-backend@ can be run:
<notextile>
<pre><code>~$ <span class="userinput">arvados-git-httpd -h</span>
-Usage of arv-git-httpd:
+Usage of arvados-git-httpd:
-address="0.0.0.0:80": Address to listen on, "host:port".
-git-command="/usr/bin/git": Path to git executable. Each authenticated request will execute this program with a single argument, "http-backend".
-repo-root="/path/to/cwd": Path to git repositories.
@@ -49,21 +248,57 @@ fatal: No REQUEST_METHOD from server
</code></pre>
</notextile>
-We recommend running @arvados-git-httpd@ under "runit":http://smarden.org/runit/ or something similar.
+h3. Enable arvados-git-httpd
-Your @run@ script should look something like this:
+Install "runit":http://smarden.org/runit/ (if it's not already installed) and configure it to run arvados-git-httpd. Update the API host to match your site.
<notextile>
-<pre><code>export ARVADOS_API_HOST=<span class="userinput">uuid_prefix</span>.your.domain
-exec sudo -u git arvados-git-httpd -address=:9001 -git-command="$(which git)" -repo-root=<span class="userinput">/var/lib/arvados/git</span> 2>&1
+<pre><code>~$ <span class="userinput">sudo apt-get install runit</span>
+~$ <span class="userinput">cd /etc/sv</span>
+/etc/sv$ <span class="userinput">sudo mkdir arvados-git-httpd; cd arvados-git-httpd</span>
+/etc/sv/arvados-git-httpd$ <span class="userinput">sudo mkdir log</span>
+/etc/sv/arvados-git-httpd$ <span class="userinput">sudo sh -c 'cat >log/run' <<'EOF'
+#!/bin/sh
+mkdir -p main
+chown git:git main
+exec chpst -u git:git svlogd -tt main
+EOF</span>
+/etc/sv/arvados-git-httpd$ <span class="userinput">sudo sh -c 'cat >run' <<'EOF'
+#!/bin/sh
+export ARVADOS_API_HOST=<b>uuid_prefix.your.domain</b>
+export GITOLITE_HTTP_HOME=/var/lib/arvados/git
+export PATH="$PATH:/var/lib/arvados/git/bin"
+exec chpst -u git:git arvados-git-httpd -address=:9001 -git-command="$(which git)" -repo-root=<b>/var/lib/arvados/git/repositories</b> 2>&1
+EOF</span>
+/etc/sv/arvados-git-httpd$ <span class="userinput">sudo chmod +x run log/run</span>
</code></pre>
</notextile>
-h3. Set up a reverse proxy with SSL support
+h3. Set up a reverse proxy to provide SSL service
+
+The arvados-git-httpd service will be accessible from anywhere on the internet, so we recommend using SSL.
-The arvados-git-httpd service will be accessible from anywhere on the internet, so we recommend using SSL for transport encryption.
+This is best achieved by putting a reverse proxy with SSL support in front of arvados-git-httpd, running on port 443 and passing requests to @arvados-git-httpd@ on port 9001 (or whichever port you used in your run script).
-This is best achieved by putting a reverse proxy with SSL support in front of arvados-git-httpd, running on port 443 and passing requests to arvados-git-httpd on port 9001 (or whatever port you chose in your run script).
+<notextile>
+<pre><code><span class="userinput">http {
+ upstream arvados-git-httpd {
+ server localhost:9001;
+ }
+ server {
+ listen *:443 ssl;
+ server_name git.uuid_prefix.example.com;
+ ssl_certificate /root/git.uuid_prefix.example.com.crt;
+ ssl_certificate_key /root/git.uuid_prefix.example.com.key;
+ location / {
+ proxy_pass http://arvados-git-httpd;
+ proxy_set_header X-Forwarded-For $remote_addr;
+ }
+ }
+}
+</span>
+</code></pre>
+</notextile>
h3. Tell the API server about the arvados-git-httpd service
diff --git a/doc/install/install-shell-server.html.textile.liquid b/doc/install/install-shell-server.html.textile.liquid
index 71e6b01..5383ad4 100644
--- a/doc/install/install-shell-server.html.textile.liquid
+++ b/doc/install/install-shell-server.html.textile.liquid
@@ -8,14 +8,14 @@ There is nothing inherently special about an Arvados shell server. It is just a
h2. Install API tokens
-Please follow the "API token guide":{{site.baseurl}}/user/reference/api-tokens.html to get API tokens for your user and install them on your shell server. We will use those tokens to test the SDKs as we install them.
+Please follow the "API token guide":../user/reference/api-tokens.html to get API tokens for your Arvados account and install them on your shell server. We will use those tokens to test the SDKs as we install them.
h2. Install the Ruby SDK and utilities
If you're using RVM:
<notextile>
-<pre><code>~$ <span class="userinput">sudo rvm-exec gem install arvados-cli</span>
+<pre><code>~$ <span class="userinput">sudo /usr/local/rvm/bin/rvm-exec default gem install arvados-cli</span>
</code></pre>
</notextile>
@@ -46,13 +46,11 @@ On Red Hat-based systems:
h2. Update Git Config
-Configure git to use the ARVADOS_API_TOKEN environment variable to authenticate to arv-git-httpd.
-
-Execute the following commands to setup the needed configuration.
+Configure git to use the ARVADOS_API_TOKEN environment variable to authenticate to arv-git-httpd. We use the @--system@ flag so it takes effect for all current and future user accounts. It does not affect git's behavior when connecting to other git servers.
<notextile>
<pre>
-<code>~$ <span class="userinput">git config 'credential.https://git.{{ site.arvados_api_host }}/.username' none</span></code>
-<code>~$ <span class="userinput">git config 'credential.https://git.{{ site.arvados_api_host }}/.helper' '!cred(){ cat >/dev/null; if [ "$1" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred'</span></code>
+<code>~$ <span class="userinput">sudo git config --system 'credential.https://git.<b>uuid_prefix.your.domain</b>/.username' none</span></code>
+<code>~$ <span class="userinput">sudo git config --system 'credential.https://git.<b>uuid_prefix.your.domain</b>/.helper' '!cred(){ cat >/dev/null; if [ "$1" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred'</span></code>
</pre>
</notextile>
diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index 758c015..6a3428c 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -59,9 +59,9 @@ ADD keep_proxy.json /root/
# Set up update-gitolite.rb
RUN mkdir /usr/local/arvados/config -p
-ADD generated/arvados-clients.yml /usr/local/arvados/config/
+ADD generated/arvados-clients.yml /usr/src/arvados/services/api/config/
ADD .gitolite.rc /usr/local/arvados/config/
-ADD update-gitolite.rb /usr/local/arvados/
+RUN ln /usr/src/arvados/services/api/script/arvados-git-sync.rb /usr/local/bin/
# Supervisor.
ADD supervisor.conf /etc/supervisor/conf.d/arvados.conf
diff --git a/docker/api/setup-gitolite.sh.in b/docker/api/setup-gitolite.sh.in
index d953fa7..023ca5d 100755
--- a/docker/api/setup-gitolite.sh.in
+++ b/docker/api/setup-gitolite.sh.in
@@ -60,7 +60,7 @@ git push
su - git -c "git clone --bare git://github.com/curoverse/arvados.git /home/git/repositories/arvados.git"
echo "ARVADOS_API_HOST_INSECURE=yes" > /etc/cron.d/gitolite-update
-echo "*/2 * * * * root /bin/bash -c 'source /etc/profile.d/rvm.sh && /usr/local/arvados/update-gitolite.rb production'" >> /etc/cron.d/gitolite-update
+echo "*/2 * * * * root /bin/bash -c 'source /etc/profile.d/rvm.sh && /usr/src/arvados/services/api/script/arvados-git-sync.rb production'" >> /etc/cron.d/gitolite-update
# Create/update the repos now
. /etc/profile.d/rvm.sh
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 409dea6..5d1402d 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -86,7 +86,7 @@ common:
# able to submit crunch jobs. To pass the test suites, put a clone
# of the arvados tree in {git_repositories_dir}/arvados.git or
# {git_repositories_dir}/arvados/.git
- git_repositories_dir: /var/lib/arvados/git
+ git_repositories_dir: /var/lib/gitolite/repositories
# This is a (bare) repository that stores commits used in jobs. When a job
# runs, the source commits are first fetched into this repository, then this
diff --git a/docker/api/update-gitolite.rb b/services/api/script/arvados-git-sync.rb
similarity index 96%
rename from docker/api/update-gitolite.rb
rename to services/api/script/arvados-git-sync.rb
index 1f178cb..3a8ed27 100755
--- a/docker/api/update-gitolite.rb
+++ b/services/api/script/arvados-git-sync.rb
@@ -21,19 +21,18 @@ DEBUG = 1
# load and merge in the environment-specific application config info
# if present, overriding base config parameters as specified
-path = File.dirname(__FILE__) + '/config/arvados-clients.yml'
+path = File.absolute_path('../../config/arvados-clients.yml', __FILE__)
if File.exists?(path) then
cp_config = YAML.load_file(path)[ENV['RAILS_ENV']]
else
- puts "Please create a\n " + File.dirname(__FILE__) + "/config/arvados-clients.yml\n file"
+ puts "Please create a\n #{path}\n file"
exit 1
end
gitolite_url = cp_config['gitolite_url']
gitolite_arvados_git_user_key = cp_config['gitolite_arvados_git_user_key']
-gitolite_tmpdir = File.join(File.absolute_path(File.dirname(__FILE__)),
- cp_config['gitolite_tmp'])
+gitolite_tmpdir = cp_config['gitolite_tmp']
gitolite_admin = File.join(gitolite_tmpdir, 'gitolite-admin')
gitolite_admin_keydir = File.join(gitolite_admin, 'keydir')
gitolite_keydir = File.join(gitolite_admin, 'keydir', 'arvados')
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list