[ARVADOS] updated: fdd4b1cb5bfbdff294d8e6c81f0c681142f1c6ea
git at public.curoverse.com
git at public.curoverse.com
Thu Apr 16 16:36:30 EDT 2015
Summary of changes:
apps/workbench/Gemfile | 2 +-
apps/workbench/Gemfile.lock | 4 +-
apps/workbench/app/assets/images/mouse-move.gif | Bin 0 -> 26082 bytes
.../app/assets/images/pipeline-running.gif | Bin 0 -> 114564 bytes
.../app/assets/javascripts/modal_pager.js | 44 +++
.../app/assets/stylesheets/application.css.scss | 4 +
.../app/controllers/actions_controller.rb | 202 ++++++------
.../app/controllers/application_controller.rb | 39 ++-
apps/workbench/app/helpers/application_helper.rb | 25 +-
apps/workbench/app/helpers/jobs_helper.rb | 4 +-
apps/workbench/app/models/arvados_base.rb | 4 +
apps/workbench/app/models/arvados_resource_list.rb | 6 +
.../app/views/application/_content.html.erb | 30 +-
.../views/application/_title_and_buttons.html.erb | 24 +-
.../_getting_started_popup.html.erb | 179 +++++++++++
apps/workbench/app/views/layouts/body.html.erb | 34 +-
.../pipeline_instances/_running_component.html.erb | 11 +-
.../_show_components_editable.html.erb | 20 ++
.../_show_components_running.html.erb | 15 +
apps/workbench/app/views/users/profile.html.erb | 13 +-
apps/workbench/config/application.default.yml | 1 +
.../test/controllers/actions_controller_test.rb | 69 ++--
.../controllers/application_controller_test.rb | 26 ++
.../test/integration/anonymous_access_test.rb | 92 ++++++
.../test/integration/application_layout_test.rb | 60 +++-
.../test/integration/report_issue_test.rb | 3 +-
.../test/integration/user_profile_test.rb | 24 +-
doc/_config.yml | 20 +-
doc/_includes/_navbar_top.liquid | 6 +-
doc/_layouts/default.html.liquid | 2 +-
doc/index.html.liquid | 32 +-
doc/user/index.html.textile.liquid | 4 +-
docker/mkimage-debootstrap.sh | 2 +-
sdk/cli/bin/crunch-job | 5 +-
sdk/go/keepclient/keepclient.go | 347 ++++++++++-----------
sdk/go/keepclient/keepclient_test.go | 269 ++++++++++++----
sdk/go/keepclient/root_sorter.go | 12 +-
sdk/go/keepclient/root_sorter_test.go | 9 +-
sdk/go/keepclient/support.go | 46 +--
sdk/python/arvados/arvfile.py | 23 +-
sdk/python/arvados/collection.py | 77 ++++-
sdk/python/arvados/keep.py | 103 ++++--
sdk/python/tests/arvados_testutil.py | 5 +-
sdk/python/tests/test_arvfile.py | 54 +++-
sdk/python/tests/test_collections.py | 59 ++--
sdk/python/tests/test_keep_client.py | 67 +++-
sdk/ruby/lib/arvados/collection.rb | 55 +++-
sdk/ruby/test/test_collection.rb | 87 +++++-
.../{zz_load_config.rb => load_config.rb} | 10 +
...preload_all_models.rb => preload_all_models.rb} | 7 +-
.../test/fixtures/api_client_authorizations.yml | 10 +-
services/api/test/fixtures/jobs.yml | 16 +
services/api/test/fixtures/logs.yml | 38 +++
services/api/test/fixtures/pipeline_instances.yml | 72 +++++
services/api/test/fixtures/users.yml | 27 +-
.../functional/arvados/v1/users_controller_test.rb | 6 +-
services/keepproxy/keepproxy.go | 220 ++++++-------
services/keepproxy/keepproxy_test.go | 10 +-
services/keepstore/keepstore.go | 2 +-
services/keepstore/pull_worker.go | 8 +-
services/keepstore/pull_worker_integration_test.go | 10 +-
services/keepstore/pull_worker_test.go | 2 +-
62 files changed, 1928 insertions(+), 729 deletions(-)
create mode 100644 apps/workbench/app/assets/images/mouse-move.gif
create mode 100644 apps/workbench/app/assets/images/pipeline-running.gif
create mode 100644 apps/workbench/app/assets/javascripts/modal_pager.js
create mode 100644 apps/workbench/app/views/getting_started/_getting_started_popup.html.erb
rename services/api/config/initializers/{zz_load_config.rb => load_config.rb} (80%)
rename services/api/config/initializers/{zz_preload_all_models.rb => preload_all_models.rb} (55%)
discards d8626f8ae89b0748bc8ac45c1050c4090a166d7c (commit)
discards d88c3bf86de3a57dc95f327290c121f7e65295a9 (commit)
discards d16fcbb426ad0e783ec615fc01b123505f5bd8fa (commit)
discards 574281cacb5f978245ef2c7551f164ccb75ef29a (commit)
discards 41b9c964f5183856b0fd2eb269a715392558c7ac (commit)
discards 6696741b436d47d76603b448f97565e7c9e2632a (commit)
via fdd4b1cb5bfbdff294d8e6c81f0c681142f1c6ea (commit)
via 602b2907114a1058ad9ddb4539376f927545113b (commit)
via 07ade15e1418ab4ca4d79b24d946cca4cc8af05b (commit)
via 80451600db3f3a1a0c8f261437ccabdce51d7a28 (commit)
via 87e5c554f2d8583d39ea347882e6d22b0a9d3afd (commit)
via b8f941d086fdcc7d882a60c59090e2827fe1adec (commit)
via 912464ad82bad38f1ce7984b6d4b19734a9816a9 (commit)
via c50acf62d8bc4a25f3fb432479fabad9b060f878 (commit)
via 1f6a187a7011becd08bb3cef8bc5c4e253900590 (commit)
via 08c4f5f03fd9afd86c4d9c43c7b8620ef9a0fade (commit)
via 33d54abd1574c16685e2d631d367a1be4e969018 (commit)
via cdecff5125a3bc09720be4d210536a183e43ce7e (commit)
via 18ac672a5c9b5416e21a3d45ed268f2edb509786 (commit)
via cd8db86a9178dd43f9582f55d9a14bde3f2e348a (commit)
via 0bff4d895d0ce312524c9d689923c9590f81f36f (commit)
via be238c74d0d3835b0a384b712d7591dc95da8c07 (commit)
via a756a90a381473adcfe8cce6a022f1733395610a (commit)
via 8e2150eb5dd257f0715c9ac050a8260279bb5d5c (commit)
via d743a56addcde9d92876d19201b9f46db42ea582 (commit)
via aafef3d22a7b225e82da5f146cb05472747a7b6e (commit)
via 6261cf9003ec37622d38a3c40d94a75eff397922 (commit)
via 1b2afc5aa599eb452a1f30e706e19b964e26cae0 (commit)
via 0f119f7707d93e90842fce3890deffc59c5e7081 (commit)
via 0d1d51b93a9009b6d4e423871886f0cbc65584c7 (commit)
via 3bc94fb2d1e5c1e351f822ca5ec8f83260e039bb (commit)
via 58e91ef73e6802978dfea0f93072d75be8ee221c (commit)
via e5b35f15d99b6371ef477408552375a762a12b38 (commit)
via 986e124a61c64ae1a31fdbc33d2da6ac061ada17 (commit)
via 3280e2dc5fd16dca63c389b931658d4420faabaf (commit)
via 3f59bec0b3be2cb8718ee310df490c7f41aa8194 (commit)
via a632845c40c3a3421914efa1617e0a886d8b1d63 (commit)
via 84f85c502498a863ea6bdc0fb6d2a7d0d4175936 (commit)
via dbdd94518d916b89f8e3a3fa0901da21a2493962 (commit)
via 3e271eb8bc19f67465a33a69dc66a27021301fae (commit)
via fe9235552e6998eb86bc2ad80716bafe7a7ffd16 (commit)
via 01dae203dd620f02b5bfa4aebf4fb217aa2817dc (commit)
via 6c4d59cb70206b1770bd64b8db204d669cb2c55a (commit)
via 93f6a595b7e69f955f2adee12ec512aa078cfdc9 (commit)
via 92b3f3edf77492c6667d785e2d7214231dd9e78a (commit)
via 9fb28073a1c7a140e44a20e0555028e9a9dbcf51 (commit)
via 3f016eae9d2431abfb8e8cc1cabdb1a494e49bf2 (commit)
via d72f3039300613487af606ce69e5b7b8b2c67027 (commit)
via cc133acd3b94cfdf6b0770f12c0a4ed6b458ef18 (commit)
via a8ef8be836b1805b4e35922ea787caad7b81f689 (commit)
via 320c333f2b49acd698b49b73ed01d32da4d15c8c (commit)
via 43aa02f5af636f874bac5ffe96cff0061bcd6a44 (commit)
via ee60fe9a20238dfac97dd988380c0149a84c372a (commit)
via ef2572886ea8693f16316cbfe537053f106a2bb5 (commit)
via 95298001afb4cb471d16bd181a8487ebe58bb0d4 (commit)
via 0acda438a00257099b07141f21ad18cf92f03355 (commit)
via 8893e8cf0dfd542508e9e45d715ee0165b249bd8 (commit)
via c17712a6fd8264e3b9fecfedc5124a9b6df00a14 (commit)
via ec8879aa56810dfc6475ad8cdc56770ff91f84f2 (commit)
via ee0411ef10f4e1c9aa207b8aa9c92bc36cb629e6 (commit)
via bd4fd5000f4f55035a0c31abb134d743888de72e (commit)
via 873226313cdc0d05a39a79a60a4fccdbf8b8c0fc (commit)
via 77b9859abb4c17355ca9db75817792733204745d (commit)
via 93ba131c85e89a3e4a2d8fc6f7dcf5d8de3818eb (commit)
via 679eef491b59a4c38a0b2914ff85e0a8c5059b2c (commit)
via c162a42573cdce83a35c54e630622544404012f7 (commit)
via 6564944612790934531a3c30e6ab2cab6f329461 (commit)
via e5658ecaa629572c5fe8e459f4f0630f28a4317f (commit)
via 6529d3fbe051c591bc22809c2dea32a5ac9eac50 (commit)
via a12cd27da6ba612f2f6ca96cd1b398b3fa621a64 (commit)
via 88f57312fe5e2226f113dc5ecb0a5edce018701d (commit)
via de7c71aac6a8e93f84d515e42859cce674eab009 (commit)
via 645b161de03001fee5663397f033a2efb17bda98 (commit)
via 6dffa00d8e284e3e5266b9d318cd6c732ae905f4 (commit)
via dcd96ef83d878c48e588a815d7793e6004c4f08c (commit)
via 02840ce904bef5542090fda327c072cdd05c32b3 (commit)
via 41bd04e7f25b0b5882e971ff7104e3f17c19bf5b (commit)
via 2af59ce37d2332db14e880aff0ba8e38fdc4781a (commit)
via 23c44c899b783a784bd9c60e3efda63cab2f6cc8 (commit)
via 9daf42fbdb868939653c6e3ca8a4fffd1cf94e31 (commit)
via 37b57a092568ba92a41ee94a74bb0c73e569a18c (commit)
via 58bd86e6912e25fcee566abd130997dd7bfc8b3c (commit)
via 6b00d3b8212e8775bb59c5f4deec4797ce44e576 (commit)
via 9e60acc645d8ee8e223398830e1ccfed3ea18e80 (commit)
via 9ae180ec18f1f397889a4531a12999942edd003a (commit)
via 8e4ed0edeadc4dfcc85666ef5c36619815dca8e2 (commit)
via 15fef38119793831a6717a98762c60aacb5d0264 (commit)
via ae2787c96b41c3844193cba3e2dddffabd98be33 (commit)
via 78e0599b0773976167ccb720fc1596236295337a (commit)
via 8dcbc531fdf79ab2e4eb29a6dad6dcd9fa103718 (commit)
via f01af635017a4988e07252f81072d0d37a889298 (commit)
via cd1f6c0fd2f9a91b88659b90a6fd9faee576a557 (commit)
via add9525f2f196e2490a6876fed590d1e139ea659 (commit)
via 3c2372f03566fb3ce53625d3e4783eaac0fc480e (commit)
via 3cc68fc8e599002f0f0cbf8e8d8b814f8c63838d (commit)
via 8ad6b6ec418f83baf6906a1fd99bc1aeb9c110b7 (commit)
via e6b8eaf06f0a9deff02ea1c7261fab02f3d75958 (commit)
via 4cb76110db20ce1e74f49d2fb7f5bb79c8c7d347 (commit)
via 2e56b93e61dc1ce0ee0d3d5e138259e073770b99 (commit)
via 70e171aa36234cc476f8056725b6269fda2f037d (commit)
via ee91e487669c650551c2da450a345ca5c1630081 (commit)
via 50843fcf14d836e82af855fa6775d409054b708d (commit)
via 5ea2ee645b5508c9c14cc2abe6cb8c2f24039c83 (commit)
via 6fe9234c96ecc16ccf5b683bed150c43842f90a0 (commit)
via a89c721eeb4115b5c6fc38071ef6192e604c4e12 (commit)
via e3c562dff0f84634c12e14c232a5ab677892ed6e (commit)
via 17379ed7eb6e143667e53097f5ada570f051efd1 (commit)
via 1f5a673014724c3444404658e65e32a6f9c562f3 (commit)
via 4108d902330bb04a70885f316700cf1da9e7d920 (commit)
via 15303f9cf99f00ccdef948e5ce593d9a3a88d21a (commit)
via 7a54e370484b9adb7479fde69665d4adcbb7331f (commit)
via b0a3771f2110b691882226c559eab736ab9aa34d (commit)
via 1f225f2c8479adcaaa918df25455b69464263b2c (commit)
via dd532a66ecc0d7ac4a90786d3a4de0c5c00c414e (commit)
via e59bc286dd018450cce0b662eeff8effe393d8ae (commit)
via 7ec30c76580397ca6b06bf1bbdb1ca9dc9f7af2f (commit)
via 663d024598ea7b6384b9e24e5ea760ecb710ecda (commit)
via b10a9487702db5d776c09bfa11f8abcb62f7419a (commit)
via 53fac68b8025db64c50c0370a16c7afaf4be7bd3 (commit)
via 5380c722ce344d712cab307d41e0f7a654a4070c (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (d8626f8ae89b0748bc8ac45c1050c4090a166d7c)
\
N -- N -- N (fdd4b1cb5bfbdff294d8e6c81f0c681142f1c6ea)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fdd4b1cb5bfbdff294d8e6c81f0c681142f1c6ea
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Apr 16 16:31:22 2015 -0400
5416: Disable repository browsing (and skip tests) if git version is suspected unreliable.
diff --git a/apps/workbench/app/controllers/repositories_controller.rb b/apps/workbench/app/controllers/repositories_controller.rb
index c5b3501..89dd96b 100644
--- a/apps/workbench/app/controllers/repositories_controller.rb
+++ b/apps/workbench/app/controllers/repositories_controller.rb
@@ -1,5 +1,8 @@
class RepositoriesController < ApplicationController
before_filter :set_share_links, if: -> { defined? @object }
+ if Repository.disable_repository_browsing?
+ before_filter :render_browsing_disabled, only: [:show_tree, :show_blob, :show_commit]
+ end
def index_pane_list
%w(recent help)
@@ -32,4 +35,10 @@ class RepositoriesController < ApplicationController
def show_commit
@commit = params[:commit]
end
+
+ protected
+
+ def render_browsing_disabled
+ render_not_found ActionController::RoutingError.new("Repository browsing features disabled")
+ end
end
diff --git a/apps/workbench/app/models/repository.rb b/apps/workbench/app/models/repository.rb
index 48c7f9e..bc9f87c 100644
--- a/apps/workbench/app/models/repository.rb
+++ b/apps/workbench/app/models/repository.rb
@@ -48,6 +48,15 @@ class Repository < ArvadosBase
subtree
end
+ # git 2.1.4 does not use credential helpers reliably, see #5416
+ def self.disable_repository_browsing?
+ return false if Rails.configuration.use_git2_despite_bug_risk
+ if @buggy_git_version.nil?
+ @buggy_git_version = /git version 2/ =~ `git version`
+ end
+ @buggy_git_version
+ end
+
protected
# refresh fetches the latest repository content into the local
diff --git a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
index cc3b4c8..ba68f17 100644
--- a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
@@ -99,7 +99,8 @@
<% # link to repo tree/file only if the repo is readable
# and the commit is a sha1
repo =
- (/^[0-9a-f]{40}$/ =~ current_component[:script_version] and
+ (not Repository.disable_repository_browsing? and
+ /^[0-9a-f]{40}$/ =~ current_component[:script_version] and
Repository.where(name: current_component[:repository]).first)
%>
<% [:script, :repository, :script_version, :supplied_script_version, :nondeterministic].each do |k| %>
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index 4061ee8..e28f76a 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -211,3 +211,9 @@ common:
# Enable response payload compression in Arvados API requests.
include_accept_encoding_header_in_api_requests: true
+
+ # Enable repository browsing even if git2 is installed. Repository
+ # browsing requires credential helpers, which do not work reliably
+ # as of git version 2.1.4. If you have git version 2.* and you want
+ # to use it anyway, change this to true.
+ use_git2_despite_bug_risk: false
diff --git a/apps/workbench/test/controllers/repositories_controller_test.rb b/apps/workbench/test/controllers/repositories_controller_test.rb
index 25bf557..852a602 100644
--- a/apps/workbench/test/controllers/repositories_controller_test.rb
+++ b/apps/workbench/test/controllers/repositories_controller_test.rb
@@ -69,6 +69,7 @@ class RepositoriesControllerTest < ActionController::TestCase
[:active, :spectator].each do |user|
test "show tree to #{user}" do
+ skip "git2 is unreliable" if Repository.disable_repository_browsing?
reset_api_fixtures_after_test false
sha1, _, _ = stub_repo_content
get :show_tree, {
@@ -85,6 +86,7 @@ class RepositoriesControllerTest < ActionController::TestCase
end
test "show commit to #{user}" do
+ skip "git2 is unreliable" if Repository.disable_repository_browsing?
reset_api_fixtures_after_test false
sha1, commit, _ = stub_repo_content
get :show_commit, {
@@ -96,6 +98,7 @@ class RepositoriesControllerTest < ActionController::TestCase
end
test "show blob to #{user}" do
+ skip "git2 is unreliable" if Repository.disable_repository_browsing?
reset_api_fixtures_after_test false
sha1, _, filedata = stub_repo_content filename: 'COPYING'
get :show_blob, {
@@ -110,6 +113,7 @@ class RepositoriesControllerTest < ActionController::TestCase
['', '/'].each do |path|
test "show tree with path '#{path}'" do
+ skip "git2 is unreliable" if Repository.disable_repository_browsing?
reset_api_fixtures_after_test false
sha1, _, _ = stub_repo_content filename: 'COPYING'
get :show_tree, {
diff --git a/apps/workbench/test/integration/repositories_browse_test.rb b/apps/workbench/test/integration/repositories_browse_test.rb
index a6a85b5..d936877 100644
--- a/apps/workbench/test/integration/repositories_browse_test.rb
+++ b/apps/workbench/test/integration/repositories_browse_test.rb
@@ -13,6 +13,7 @@ class RepositoriesTest < ActionDispatch::IntegrationTest
end
test "browse repository from jobs#show" do
+ skip "git2 is unreliable" if Repository.disable_repository_browsing?
sha1 = api_fixture('jobs')['running']['script_version']
_, fakecommit, fakefile =
stub_repo_content sha1: sha1, filename: 'crunch_scripts/hash'
@@ -36,6 +37,7 @@ class RepositoriesTest < ActionDispatch::IntegrationTest
end
test "browse using arv-git-http" do
+ skip "git2 is unreliable" if Repository.disable_repository_browsing?
repo = api_fixture('repositories')['foo']
portfile =
File.expand_path('../../../../../tmp/arv-git-httpd-ssl.port', __FILE__)
commit 602b2907114a1058ad9ddb4539376f927545113b
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Apr 16 15:52:20 2015 -0400
5416: Use http://foo:bar@host:port/ instead of credential helper.
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index ef16acb..df2d8d6 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -52,7 +52,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
w.WriteHeader(statusCode)
w.Write([]byte(statusText))
}
- log.Println(quoteStrings(r.RemoteAddr, username, password, wroteStatus, statusText, repoName, r.URL.Path)...)
+ log.Println(quoteStrings(r.RemoteAddr, username, password, wroteStatus, statusText, repoName, r.Method, r.URL.Path)...)
}()
// HTTP request username is logged, but unused. Password is an
@@ -87,7 +87,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
arv.ApiToken = password
reposFound := arvadosclient.Dict{}
if err := arv.List("repositories", arvadosclient.Dict{
- "filters": [][]string{[]string{"name", "=", repoName}},
+ "filters": [][]string{{"name", "=", repoName}},
}, &reposFound); err != nil {
statusCode, statusText = http.StatusInternalServerError, err.Error()
return
diff --git a/services/arv-git-httpd/server_test.go b/services/arv-git-httpd/server_test.go
index 82d71ae..d773dd9 100644
--- a/services/arv-git-httpd/server_test.go
+++ b/services/arv-git-httpd/server_test.go
@@ -14,6 +14,12 @@ import (
var _ = check.Suite(&IntegrationSuite{})
+const (
+ spectatorToken = "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu"
+ activeToken = "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi"
+ anonymousToken = "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi"
+)
+
// IntegrationSuite tests need an API server and an arv-git-httpd server
type IntegrationSuite struct {
tmpRepoRoot string
@@ -23,55 +29,43 @@ type IntegrationSuite struct {
func (s *IntegrationSuite) TestPathVariants(c *check.C) {
s.makeArvadosRepo(c)
- // Spectator token
- os.Setenv("ARVADOS_API_TOKEN", "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu")
for _, repo := range []string{"active/foo.git", "active/foo/.git", "arvados.git", "arvados/.git"} {
- err := s.runGit(c, "fetch", repo)
+ err := s.runGit(c, spectatorToken, "fetch", repo)
c.Assert(err, check.Equals, nil)
}
}
func (s *IntegrationSuite) TestReadonly(c *check.C) {
- // Spectator token
- os.Setenv("ARVADOS_API_TOKEN", "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu")
- err := s.runGit(c, "fetch", "active/foo.git")
+ err := s.runGit(c, spectatorToken, "fetch", "active/foo.git")
c.Assert(err, check.Equals, nil)
- err = s.runGit(c, "push", "active/foo.git", "master:newbranchfail")
+ err = s.runGit(c, spectatorToken, "push", "active/foo.git", "master:newbranchfail")
c.Assert(err, check.ErrorMatches, `.*HTTP code = 403.*`)
_, err = os.Stat(s.tmpRepoRoot + "/zzzzz-s0uqq-382brsig8rp3666/.git/refs/heads/newbranchfail")
c.Assert(err, check.FitsTypeOf, &os.PathError{})
}
func (s *IntegrationSuite) TestReadwrite(c *check.C) {
- // Active user token
- os.Setenv("ARVADOS_API_TOKEN", "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi")
- err := s.runGit(c, "fetch", "active/foo.git")
+ err := s.runGit(c, activeToken, "fetch", "active/foo.git")
c.Assert(err, check.Equals, nil)
- err = s.runGit(c, "push", "active/foo.git", "master:newbranch")
+ err = s.runGit(c, activeToken, "push", "active/foo.git", "master:newbranch")
c.Assert(err, check.Equals, nil)
_, err = os.Stat(s.tmpRepoRoot + "/zzzzz-s0uqq-382brsig8rp3666/.git/refs/heads/newbranch")
c.Assert(err, check.Equals, nil)
}
func (s *IntegrationSuite) TestNonexistent(c *check.C) {
- // Spectator token
- os.Setenv("ARVADOS_API_TOKEN", "zw2f4gwx8hw8cjre7yp6v1zylhrhn3m5gvjq73rtpwhmknrybu")
- err := s.runGit(c, "fetch", "thisrepodoesnotexist.git")
+ err := s.runGit(c, spectatorToken, "fetch", "thisrepodoesnotexist.git")
c.Assert(err, check.ErrorMatches, `.* not found.*`)
}
func (s *IntegrationSuite) TestMissingGitdirReadableRepository(c *check.C) {
- // Active user token
- os.Setenv("ARVADOS_API_TOKEN", "3kg6k6lzmp9kj5cpkcoxie963cmvjahbt2fod9zru30k1jqdmi")
- err := s.runGit(c, "fetch", "active/foo2.git")
+ err := s.runGit(c, activeToken, "fetch", "active/foo2.git")
c.Assert(err, check.ErrorMatches, `.* not found.*`)
}
func (s *IntegrationSuite) TestNoPermission(c *check.C) {
- // Anonymous token
- os.Setenv("ARVADOS_API_TOKEN", "4kg6k6lzmp9kj4cpkcoxie964cmvjahbt4fod9zru44k4jqdmi")
for _, repo := range []string{"active/foo.git", "active/foo/.git"} {
- err := s.runGit(c, "fetch", repo)
+ err := s.runGit(c, anonymousToken, "fetch", repo)
c.Assert(err, check.ErrorMatches, `.* not found.*`)
}
}
@@ -107,18 +101,7 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
// Clear ARVADOS_API_TOKEN after starting up the server, to
// make sure arv-git-httpd doesn't use it.
- os.Setenv("ARVADOS_API_TOKEN", "")
-
- _, err = exec.Command("git", "config",
- "--file", s.tmpWorkdir+"/.git/config",
- "credential.http://"+s.testServer.Addr+"/.helper",
- "!cred(){ echo password=$ARVADOS_API_TOKEN; };cred").Output()
- c.Assert(err, check.Equals, nil)
- _, err = exec.Command("git", "config",
- "--file", s.tmpWorkdir+"/.git/config",
- "credential.http://"+s.testServer.Addr+"/.username",
- "none").Output()
- c.Assert(err, check.Equals, nil)
+ os.Setenv("ARVADOS_API_TOKEN", "unused-token-placates-client-library")
}
func (s *IntegrationSuite) TearDownTest(c *check.C) {
@@ -137,14 +120,14 @@ func (s *IntegrationSuite) TearDownTest(c *check.C) {
}
}
-func (s *IntegrationSuite) runGit(c *check.C, gitCmd, repo string, args ...string) error {
+func (s *IntegrationSuite) runGit(c *check.C, token, gitCmd, repo string, args ...string) error {
cwd, err := os.Getwd()
c.Assert(err, check.Equals, nil)
defer os.Chdir(cwd)
os.Chdir(s.tmpWorkdir)
gitargs := append([]string{
- gitCmd, "http://" + s.testServer.Addr + "/" + repo,
+ gitCmd, "http://none:" + token + "@" + s.testServer.Addr + "/" + repo,
}, args...)
cmd := exec.Command("git", gitargs...)
w, err := cmd.StdinPipe()
commit 07ade15e1418ab4ca4d79b24d946cca4cc8af05b
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 8 11:35:56 2015 -0400
5416: Do not override git urls for remote hosted repos.
diff --git a/services/api/app/models/repository.rb b/services/api/app/models/repository.rb
index d2da6ea..e83ac41 100644
--- a/services/api/app/models/repository.rb
+++ b/services/api/app/models/repository.rb
@@ -20,11 +20,12 @@ class Repository < ArvadosModel
end
def push_url
- if Rails.configuration.git_host
- "git@%s:%s.git" % [Rails.configuration.git_host, name]
- else
- "git at git.%s.arvadosapi.com:%s.git" % [Rails.configuration.uuid_prefix, name]
+ prefix = new_record? ? Rails.configuration.uuid_prefix : uuid[0,5]
+ if prefix == Rails.configuration.uuid_prefix
+ host = Rails.configuration.git_host
end
+ host ||= "git.%s.arvadosapi.com" % prefix
+ "git@%s:%s.git" % [host, name]
end
def fetch_url
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 5ffac65..e54fa65 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -58,9 +58,9 @@ common:
# logic for deciding on a hostname.
host: false
- # If not false, this is the hostname that will be used to generate fetch_url
- # and push_url for git repositories. By default, this will be
- # git.(uuid_prefix).arvadosapi.com
+ # If not false, this is the hostname that will be used to generate
+ # fetch_url and push_url for locally hosted git repositories. By
+ # default, this is git.(uuid_prefix).arvadosapi.com
git_host: false
# If this is not false, HTML requests at the API server's root URL
diff --git a/services/api/test/unit/repository_test.rb b/services/api/test/unit/repository_test.rb
index 5acef1b..288e118 100644
--- a/services/api/test/unit/repository_test.rb
+++ b/services/api/test/unit/repository_test.rb
@@ -108,6 +108,7 @@ class RepositoryTest < ActiveSupport::TestCase
test "fetch_url" do
repo = new_repo(:active, name: "active/fetchtest")
+ repo.save
assert_equal(default_git_url("fetchtest", "active"), repo.fetch_url)
end
@@ -115,11 +116,13 @@ class RepositoryTest < ActiveSupport::TestCase
set_user_from_auth :admin
repo = Repository.new(owner_uuid: users(:system_user).uuid,
name: "fetchtest")
+ repo.save
assert_equal(default_git_url("fetchtest"), repo.fetch_url)
end
test "push_url" do
repo = new_repo(:active, name: "active/pushtest")
+ repo.save
assert_equal(default_git_url("pushtest", "active"), repo.push_url)
end
@@ -127,6 +130,7 @@ class RepositoryTest < ActiveSupport::TestCase
set_user_from_auth :admin
repo = Repository.new(owner_uuid: users(:system_user).uuid,
name: "pushtest")
+ repo.save
assert_equal(default_git_url("pushtest"), repo.push_url)
end
commit 80451600db3f3a1a0c8f261437ccabdce51d7a28
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 8 10:22:05 2015 -0400
5416: Run arv-git-httpd and nginx ssl proxy in test suite.
diff --git a/apps/workbench/test/integration/jobs_test.rb b/apps/workbench/test/integration/jobs_test.rb
index 29bccd9..7a6c540 100644
--- a/apps/workbench/test/integration/jobs_test.rb
+++ b/apps/workbench/test/integration/jobs_test.rb
@@ -98,7 +98,7 @@ class JobsTest < ActionDispatch::IntegrationTest
# that the correct script version is mentioned in the
# Fiddlesticks error message.
if expect_options && use_latest
- assert_text "Script version #{job['supplied_script_version']} does not resolve to a commit"
+ assert_text "077ba2ad3ea24a929091a9e6ce545c93199b8e57"
else
assert_text "Script version #{job['script_version']} does not resolve to a commit"
end
diff --git a/apps/workbench/test/integration/repositories_browse_test.rb b/apps/workbench/test/integration/repositories_browse_test.rb
index 147bf46..a6a85b5 100644
--- a/apps/workbench/test/integration/repositories_browse_test.rb
+++ b/apps/workbench/test/integration/repositories_browse_test.rb
@@ -34,4 +34,20 @@ class RepositoriesTest < ActionDispatch::IntegrationTest
click_on sha1
assert_text fakecommit
end
+
+ test "browse using arv-git-http" do
+ repo = api_fixture('repositories')['foo']
+ portfile =
+ File.expand_path('../../../../../tmp/arv-git-httpd-ssl.port', __FILE__)
+ gitsslport = File.read(portfile)
+ Repository.any_instance.
+ stubs(:http_fetch_url).
+ returns "https://localhost:#{gitsslport}/#{repo['name']}.git"
+ commit_sha1 = '1de84a854e2b440dc53bf42f8548afa4c17da332'
+ visit page_with_token('active', "/repositories/#{repo['uuid']}/commit/#{commit_sha1}")
+ assert_text "Date: Tue Mar 18 15:55:28 2014 -0400"
+ visit page_with_token('active', "/repositories/#{repo['uuid']}/tree/#{commit_sha1}")
+ assert_selector "tbody td a", "foo"
+ assert_text "12 bytes"
+ end
end
diff --git a/apps/workbench/test/test_helper.rb b/apps/workbench/test/test_helper.rb
index 4ab162c..61771f9 100644
--- a/apps/workbench/test/test_helper.rb
+++ b/apps/workbench/test/test_helper.rb
@@ -137,7 +137,7 @@ class ApiServerForTests
@main_process_pid = $$
@@server_is_running = false
- def check_call *args
+ def check_output *args
output = nil
Bundler.with_clean_env do
output = IO.popen *args do |io|
@@ -153,7 +153,12 @@ class ApiServerForTests
def run_test_server
env_script = nil
Dir.chdir PYTHON_TESTS_DIR do
- env_script = check_call %w(python ./run_test_server.py start --auth admin)
+ # These are no-ops if we're running within run-tests.sh (except
+ # that we do get a useful env_script back from "start", even
+ # though it doesn't need to start up a new server).
+ env_script = check_output %w(python ./run_test_server.py start --auth admin)
+ check_output %w(python ./run_test_server.py start_arv-git-httpd)
+ check_output %w(python ./run_test_server.py start_nginx)
end
test_env = {}
env_script.each_line do |line|
@@ -169,8 +174,10 @@ class ApiServerForTests
def stop_test_server
Dir.chdir PYTHON_TESTS_DIR do
- # This is a no-op if we're running within run-tests.sh
- check_call %w(python ./run_test_server.py stop)
+ # These are no-ops if we're running within run-tests.sh
+ check_output %w(python ./run_test_server.py stop_nginx)
+ check_output %w(python ./run_test_server.py stop_arv-git-httpd)
+ check_output %w(python ./run_test_server.py stop)
end
@@server_is_running = false
end
@@ -196,7 +203,7 @@ class ApiServerForTests
def run_rake_task task_name, arg_string
Dir.chdir ARV_API_SERVER_DIR do
- check_call ['bundle', 'exec', 'rake', "#{task_name}[#{arg_string}]"]
+ check_output ['bundle', 'exec', 'rake', "#{task_name}[#{arg_string}]"]
end
end
end
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
new file mode 100644
index 0000000..6196605
--- /dev/null
+++ b/sdk/python/tests/nginx.conf
@@ -0,0 +1,31 @@
+daemon off;
+error_log stderr info; # Yes, must be specified here _and_ cmdline
+events {
+}
+http {
+ access_log /dev/stderr combined;
+ upstream arv-git-http {
+ server localhost:{{GITPORT}};
+ }
+ server {
+ listen *:{{GITSSLPORT}} ssl default_server;
+ server_name _;
+ ssl_certificate {{SSLCERT}};
+ ssl_certificate_key {{SSLKEY}};
+ location / {
+ proxy_pass http://arv-git-http;
+ }
+ }
+ upstream keepproxy {
+ server localhost:{{KEEPPROXYPORT}};
+ }
+ server {
+ listen *:{{KEEPPROXYSSLPORT}} ssl default_server;
+ server_name _;
+ ssl_certificate {{SSLCERT}};
+ ssl_certificate_key {{SSLKEY}};
+ location / {
+ proxy_pass http://keepproxy;
+ }
+ }
+}
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index b9502f0..a84fb22 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -41,6 +41,7 @@ if not os.path.exists(TEST_TMPDIR):
os.mkdir(TEST_TMPDIR)
my_api_host = None
+_cached_config = {}
def find_server_pid(PID_PATH, wait=10):
now = time.time()
@@ -178,6 +179,13 @@ def run(leave_running_atexit=False):
'-subj', '/CN=0.0.0.0'],
stdout=sys.stderr)
+ # Install the git repository fixtures.
+ gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
+ gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
+ if not os.path.isdir(gitdir):
+ os.makedirs(gitdir)
+ subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
+
port = find_available_port()
env = os.environ.copy()
env['RAILS_ENV'] = 'test'
@@ -258,13 +266,14 @@ def _start_keep(n, keep_args):
keep_cmd = ["keepstore",
"-volumes={}".format(keep0),
"-listen=:{}".format(port),
- "-pid={}".format("{}/keep{}.pid".format(TEST_TMPDIR, n))]
+ "-pid="+_pidfile('keep{}'.format(n))]
for arg, val in keep_args.iteritems():
keep_cmd.append("{}={}".format(arg, val))
- kp0 = subprocess.Popen(keep_cmd)
- with open("{}/keep{}.pid".format(TEST_TMPDIR, n), 'w') as f:
+ kp0 = subprocess.Popen(
+ keep_cmd, stdin=open('/dev/null'), stdout=sys.stderr)
+ with open(_pidfile('keep{}'.format(n)), 'w') as f:
f.write(str(kp0.pid))
with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'w') as f:
@@ -307,7 +316,7 @@ def run_keep(blob_signing_key=None, enforce_permissions=False):
}).execute()
def _stop_keep(n):
- kill_server_pid("{}/keep{}.pid".format(TEST_TMPDIR, n), 0)
+ kill_server_pid(_pidfile('keep{}'.format(n)), 0)
if os.path.exists("{}/keep{}.volume".format(TEST_TMPDIR, n)):
with open("{}/keep{}.volume".format(TEST_TMPDIR, n), 'r') as r:
shutil.rmtree(r.read(), True)
@@ -320,6 +329,8 @@ def stop_keep():
_stop_keep(1)
def run_keep_proxy():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
stop_keep_proxy()
admin_token = auth_token('admin')
@@ -328,9 +339,9 @@ def run_keep_proxy():
env['ARVADOS_API_TOKEN'] = admin_token
kp = subprocess.Popen(
['keepproxy',
- '-pid={}/keepproxy.pid'.format(TEST_TMPDIR),
+ '-pid='+_pidfile('keepproxy'),
'-listen=:{}'.format(port)],
- env=env)
+ env=env, stdin=open('/dev/null'), stdout=sys.stderr)
api = arvados.api(
version='v1',
@@ -347,9 +358,100 @@ def run_keep_proxy():
'service_ssl_flag': False,
}}).execute()
os.environ["ARVADOS_KEEP_PROXY"] = "http://localhost:{}".format(port)
+ _setport('keepproxy', port)
def stop_keep_proxy():
- kill_server_pid(os.path.join(TEST_TMPDIR, "keepproxy.pid"), 0)
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ kill_server_pid(_pidfile('keepproxy'), wait=0)
+
+def run_arv_git_httpd():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ stop_arv_git_httpd()
+
+ gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
+ gitport = find_available_port()
+ env = os.environ.copy()
+ del env['ARVADOS_API_TOKEN']
+ agh = subprocess.Popen(
+ ['arv-git-httpd',
+ '-repo-root='+gitdir+'/test',
+ '-address=:'+str(gitport)],
+ env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ with open(_pidfile('arv-git-httpd'), 'w') as f:
+ f.write(str(agh.pid))
+ _setport('arv-git-httpd', gitport)
+
+def stop_arv_git_httpd():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ kill_server_pid(_pidfile('arv-git-httpd'), wait=0)
+
+def run_nginx():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ nginxconf = {}
+ nginxconf['KEEPPROXYPORT'] = _getport('keepproxy')
+ nginxconf['KEEPPROXYSSLPORT'] = find_available_port()
+ nginxconf['GITPORT'] = _getport('arv-git-httpd')
+ nginxconf['GITSSLPORT'] = find_available_port()
+ nginxconf['SSLCERT'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.pem')
+ nginxconf['SSLKEY'] = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'self-signed.key')
+
+ conftemplatefile = os.path.join(MY_DIRNAME, 'nginx.conf')
+ conffile = os.path.join(TEST_TMPDIR, 'nginx.conf')
+ with open(conffile, 'w') as f:
+ f.write(re.sub(
+ r'{{([A-Z]+)}}',
+ lambda match: str(nginxconf.get(match.group(1))),
+ open(conftemplatefile).read()))
+
+ env = os.environ.copy()
+ env['PATH'] = env['PATH']+':/sbin:/usr/sbin:/usr/local/sbin'
+ nginx = subprocess.Popen(
+ ['nginx',
+ '-g', 'error_log stderr info;',
+ '-g', 'pid '+_pidfile('nginx')+';',
+ '-c', conffile],
+ env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT'])
+ _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT'])
+
+def stop_nginx():
+ if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ:
+ return
+ kill_server_pid(_pidfile('nginx'), wait=0)
+
+def _pidfile(program):
+ return os.path.join(TEST_TMPDIR, program + '.pid')
+
+def _portfile(program):
+ return os.path.join(TEST_TMPDIR, program + '.port')
+
+def _setport(program, port):
+ with open(_portfile(program), 'w') as f:
+ f.write(str(port))
+
+# Returns 9 if program is not up.
+def _getport(program):
+ try:
+ return int(open(_portfile(program)).read())
+ except IOError:
+ return 9
+
+def _apiconfig(key):
+ if _cached_config:
+ return _cached_config[key]
+ def _load(f):
+ return yaml.load(os.path.join(SERVICES_SRC_DIR, 'api', 'config', f))
+ cdefault = _load('application.default.yml')
+ csite = _load('application.yml')
+ _cached_config = {}
+ for section in [cdefault.get('common',{}), cdefault.get('test',{}),
+ csite.get('common',{}), csite.get('test',{})]:
+ _cached_config.update(section)
+ return _cached_config[key]
def fixture(fix):
'''load a fixture yaml file'''
@@ -460,5 +562,13 @@ if __name__ == "__main__":
run_keep_proxy()
elif args.action == 'stop_keep_proxy':
stop_keep_proxy()
+ elif args.action == 'start_arv-git-httpd':
+ run_arv_git_httpd()
+ elif args.action == 'stop_arv-git-httpd':
+ stop_arv_git_httpd()
+ elif args.action == 'start_nginx':
+ run_nginx()
+ elif args.action == 'stop_nginx':
+ stop_nginx()
else:
print("Unrecognized action '{}'. Actions are: {}.".format(args.action, actions))
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 1696e2c..5ffac65 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -45,6 +45,7 @@ test:
blob_signing_key: zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc
user_profile_notification_address: arvados at example.com
workbench_address: https://localhost:3001/
+ git_repositories_dir: <%= Rails.root.join 'tmp', 'git', 'test' %>
common:
# The prefix used for all database identifiers to identify the record as
diff --git a/services/api/test/helpers/git_test_helper.rb b/services/api/test/helpers/git_test_helper.rb
index 67e99c1..7b1fb4e 100644
--- a/services/api/test/helpers/git_test_helper.rb
+++ b/services/api/test/helpers/git_test_helper.rb
@@ -14,9 +14,13 @@ require 'tmpdir'
module GitTestHelper
def self.included base
base.setup do
- @tmpdir = Dir.mktmpdir()
- system("tar", "-xC", @tmpdir, "-f", "test/test.git.tar")
- Rails.configuration.git_repositories_dir = "#{@tmpdir}/test"
+ # Extract the test repository data into the default test
+ # environment's Rails.configuration.git_repositories_dir. (We
+ # don't use that config setting here, though: it doesn't seem
+ # worth the risk of stepping on a real git repo root.)
+ @tmpdir = Rails.root.join 'tmp', 'git'
+ FileUtils.mkdir_p @tmpdir
+ system("tar", "-xC", @tmpdir.to_s, "-f", "test/test.git.tar")
Commit.refresh_repositories
end
commit 87e5c554f2d8583d39ea347882e6d22b0a9d3afd
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 1 18:34:38 2015 -0400
5416: Browse git repository contents in workbench.
diff --git a/apps/workbench/app/controllers/repositories_controller.rb b/apps/workbench/app/controllers/repositories_controller.rb
index d32c92a..c5b3501 100644
--- a/apps/workbench/app/controllers/repositories_controller.rb
+++ b/apps/workbench/app/controllers/repositories_controller.rb
@@ -16,4 +16,20 @@ class RepositoriesController < ApplicationController
panes.delete('Attributes') if !current_user.is_admin
panes
end
+
+ def show_tree
+ @commit = params[:commit]
+ @path = params[:path] || ''
+ @subtree = @object.ls_subtree @commit, @path.chomp('/')
+ end
+
+ def show_blob
+ @commit = params[:commit]
+ @path = params[:path]
+ @blobdata = @object.cat_file @commit, @path
+ end
+
+ def show_commit
+ @commit = params[:commit]
+ end
end
diff --git a/apps/workbench/app/models/repository.rb b/apps/workbench/app/models/repository.rb
index b062dda..48c7f9e 100644
--- a/apps/workbench/app/models/repository.rb
+++ b/apps/workbench/app/models/repository.rb
@@ -12,4 +12,105 @@ class Repository < ArvadosBase
[]
end
end
+
+ def show commit_sha1
+ refresh
+ run_git 'show', commit_sha1
+ end
+
+ def cat_file commit_sha1, path
+ refresh
+ run_git 'cat-file', 'blob', commit_sha1 + ':' + path
+ end
+
+ def ls_tree_lr commit_sha1
+ refresh
+ run_git 'ls-tree', '-l', '-r', commit_sha1
+ end
+
+ # subtree returns a list of files under the given path at the
+ # specified commit. Results are returned as an array of file nodes,
+ # where each file node is an array [file mode, blob sha1, file size
+ # in bytes, path relative to the given directory]. If the path is
+ # not found, [] is returned.
+ def ls_subtree commit, path
+ path = path.chomp '/'
+ subtree = []
+ ls_tree_lr(commit).each_line do |line|
+ mode, type, sha1, size, filepath = line.split
+ next if type != 'blob'
+ if filepath[0,path.length] == path and
+ (path == '' or filepath[path.length] == '/')
+ subtree << [mode.to_i(8), sha1, size.to_i,
+ filepath[path.length,filepath.length]]
+ end
+ end
+ subtree
+ end
+
+ protected
+
+ # refresh fetches the latest repository content into the local
+ # cache. It is a no-op if it has already been run on this object:
+ # this (pretty much) avoids doing more than one remote git operation
+ # per Workbench request.
+ def refresh
+ run_git 'fetch', http_fetch_url, '+*:*' unless @fresh
+ @fresh = true
+ end
+
+ # http_fetch_url returns an http:// or https:// fetch-url which can
+ # accept arvados API token authentication. The API server currently
+ # advertises SSH fetch-urls, which work for users with SSH keys but
+ # are useless for fetching repository content into workbench itself.
+ def http_fetch_url
+ "https://git.#{uuid[0,5]}.arvadosapi.com/#{name}.git"
+ end
+
+ # run_git sets up the ARVADOS_API_TOKEN environment variable,
+ # creates a local git directory for this repository if necessary,
+ # executes "git --git-dir localgitdir {args to run_git}", and
+ # returns the output. It raises GitCommandError if git exits
+ # non-zero.
+ def run_git *gitcmd
+ if not @workdir
+ workdir = File.expand_path uuid+'.git', Rails.configuration.repository_cache
+ if not File.exists? workdir
+ FileUtils.mkdir_p Rails.configuration.repository_cache
+ [['git', 'init', '--bare', workdir],
+ ].each do |cmd|
+ system *cmd
+ raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0
+ end
+ end
+ @workdir = workdir
+ end
+ [['git', '--git-dir', @workdir, 'config', '--local',
+ "credential.#{http_fetch_url}.username", 'none'],
+ ['git', '--git-dir', @workdir, 'config', '--local',
+ "credential.#{http_fetch_url}.helper",
+ '!token(){ echo password="$ARVADOS_API_TOKEN"; }; token'],
+ ['git', '--git-dir', @workdir, 'config', '--local',
+ 'http.sslVerify',
+ Rails.configuration.arvados_insecure_https ? 'false' : 'true'],
+ ].each do |cmd|
+ system *cmd
+ raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0
+ end
+ env = {}.
+ merge(ENV).
+ merge('ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token])
+ cmd = ['git', '--git-dir', @workdir] + gitcmd
+ io = IO.popen(env, cmd, err: [:child, :out])
+ output = io.read
+ io.close
+ # "If [io] is opened by IO.popen, close sets $?." --ruby 2.2.1 docs
+ unless $?.exitstatus == 0
+ raise GitCommandError.new("`git #{gitcmd.join ' '}` #{$?}: #{output}")
+ end
+ output
+ end
+
+ class GitCommandError < StandardError
+ end
end
diff --git a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
index 1a9cb35..cc3b4c8 100644
--- a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
@@ -96,6 +96,12 @@
<div class="row">
<div class="col-md-6">
<table>
+ <% # link to repo tree/file only if the repo is readable
+ # and the commit is a sha1
+ repo =
+ (/^[0-9a-f]{40}$/ =~ current_component[:script_version] and
+ Repository.where(name: current_component[:repository]).first)
+ %>
<% [:script, :repository, :script_version, :supplied_script_version, :nondeterministic].each do |k| %>
<tr>
<td style="padding-right: 1em">
@@ -104,6 +110,12 @@
<td>
<% if current_component[k].nil? %>
(none)
+ <% elsif repo and k == :repository %>
+ <%= link_to current_component[k], show_repository_tree_path(id: repo.uuid, commit: current_component[:script_version], path: '/') %>
+ <% elsif repo and k == :script %>
+ <%= link_to current_component[k], show_repository_blob_path(id: repo.uuid, commit: current_component[:script_version], path: 'crunch_scripts/'+current_component[:script]) %>
+ <% elsif repo and k == :script_version %>
+ <%= link_to current_component[k], show_repository_commit_path(id: repo.uuid, commit: current_component[:script_version]) %>
<% else %>
<%= current_component[k] %>
<% end %>
diff --git a/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb b/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb
new file mode 100644
index 0000000..736c187
--- /dev/null
+++ b/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb
@@ -0,0 +1,13 @@
+<div class="pull-right">
+ <span class="deemphasize">Browsing <%= @object.name %> repository at commit</span>
+ <%= link_to(@commit, show_repository_commit_path(id: @object.uuid, commit: @commit), title: 'show commit message') %>
+</div>
+<p>
+ <%= link_to(@object.name, show_repository_tree_path(id: @object.uuid, commit: @commit, path: '/'), title: 'show root directory of source tree') %>
+ <% parents = ''
+ (@path || '').split('/').each do |pathpart|
+ parents = parents + pathpart + '/'
+ %>
+ / <%= link_to pathpart, show_repository_tree_path(id: @object.uuid, commit: @commit, path: parents) %>
+ <% end %>
+</p>
diff --git a/apps/workbench/app/views/repositories/show_blob.html.erb b/apps/workbench/app/views/repositories/show_blob.html.erb
new file mode 100644
index 0000000..acc34d1
--- /dev/null
+++ b/apps/workbench/app/views/repositories/show_blob.html.erb
@@ -0,0 +1,13 @@
+<%= render partial: 'repository_breadcrumbs' %>
+
+<% if not @blobdata.valid_encoding? %>
+ <div class="alert alert-warning">
+ <p>
+ This file has an invalid text encoding, so it can't be shown
+ here. (This probably just means it's a binary file, not a text
+ file.)
+ </p>
+ </div>
+<% else %>
+ <pre><%= @blobdata %></pre>
+<% end %>
diff --git a/apps/workbench/app/views/repositories/show_commit.html.erb b/apps/workbench/app/views/repositories/show_commit.html.erb
new file mode 100644
index 0000000..3690be6
--- /dev/null
+++ b/apps/workbench/app/views/repositories/show_commit.html.erb
@@ -0,0 +1,3 @@
+<%= render partial: 'repository_breadcrumbs' %>
+
+<pre><%= @object.show @commit %></pre>
diff --git a/apps/workbench/app/views/repositories/show_tree.html.erb b/apps/workbench/app/views/repositories/show_tree.html.erb
new file mode 100644
index 0000000..4e2fcec
--- /dev/null
+++ b/apps/workbench/app/views/repositories/show_tree.html.erb
@@ -0,0 +1,40 @@
+<%= render partial: 'repository_breadcrumbs' %>
+
+<table class="table table-condensed table-hover">
+ <thead>
+ <tr>
+ <th>File</th>
+ <th class="data-size">Size</th>
+ </tr>
+ </thead>
+ <tbody>
+ <% @subtree.each do |mode, sha1, size, subpath| %>
+ <tr>
+ <td>
+ <span style="opacity: 0.6">
+ <% pathparts = subpath.sub(/^\//, '').split('/')
+ basename = pathparts.pop
+ parents = @path
+ pathparts.each do |pathpart| %>
+ <% parents = parents + '/' + pathpart %>
+ <%= link_to pathpart, url_for(path: parents) %>
+ /
+ <% end %>
+ </span>
+ <%= link_to basename, url_for(action: :show_blob, path: parents + '/' + basename) %>
+ </td>
+ <td class="data-size">
+ <%= human_readable_bytes_html(size) %>
+ </td>
+ </tr>
+ <% end %>
+ <% if @subtree.empty? %>
+ <tr>
+ <td>
+ No files found.
+ </td>
+ </tr>
+ <% end %>
+ </tbody>
+ <tfoot></tfoot>
+</table>
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index 8aeacf4..4061ee8 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -139,8 +139,14 @@ common:
default_openid_prefix: https://www.google.com/accounts/o8/id
send_user_setup_notification_email: true
- # Set user_profile_form_fields to enable and configure the user profile page.
- # Default is set to false. A commented setting with full description is provided below.
+ # Scratch directory used by the remote repository browsing
+ # feature. If it doesn't exist, it (and any missing parents) will be
+ # created using mkdir_p.
+ repository_cache: <%= File.expand_path 'tmp/git', Rails.root %>
+
+ # Set user_profile_form_fields to enable and configure the user
+ # profile page. Default is set to false. A commented example with
+ # full description is provided below.
user_profile_form_fields: false
# Below is a sample setting of user_profile_form_fields config parameter.
diff --git a/apps/workbench/config/routes.rb b/apps/workbench/config/routes.rb
index 7ed02e7..44d7ded 100644
--- a/apps/workbench/config/routes.rb
+++ b/apps/workbench/config/routes.rb
@@ -26,6 +26,11 @@ ArvadosWorkbench::Application.routes.draw do
resources :repositories do
post 'share_with', on: :member
end
+ # {format: false} prevents rails from treating "foo.png" as foo?format=png
+ get '/repositories/:id/tree/:commit' => 'repositories#show_tree'
+ get '/repositories/:id/tree/:commit/*path' => 'repositories#show_tree', as: :show_repository_tree, format: false
+ get '/repositories/:id/blob/:commit/*path' => 'repositories#show_blob', as: :show_repository_blob, format: false
+ get '/repositories/:id/commit/:commit' => 'repositories#show_commit', as: :show_repository_commit
match '/logout' => 'sessions#destroy', via: [:get, :post]
get '/logged_out' => 'sessions#index'
resources :users do
diff --git a/apps/workbench/test/controllers/repositories_controller_test.rb b/apps/workbench/test/controllers/repositories_controller_test.rb
index f95bb77..25bf557 100644
--- a/apps/workbench/test/controllers/repositories_controller_test.rb
+++ b/apps/workbench/test/controllers/repositories_controller_test.rb
@@ -1,7 +1,9 @@
require 'test_helper'
+require 'helpers/repository_stub_helper'
require 'helpers/share_object_helper'
class RepositoriesControllerTest < ActionController::TestCase
+ include RepositoryStubHelper
include ShareObjectHelper
[
@@ -62,4 +64,61 @@ class RepositoriesControllerTest < ActionController::TestCase
end
end
end
+
+ ### Browse repository content
+
+ [:active, :spectator].each do |user|
+ test "show tree to #{user}" do
+ reset_api_fixtures_after_test false
+ sha1, _, _ = stub_repo_content
+ get :show_tree, {
+ id: api_fixture('repositories')['foo']['uuid'],
+ commit: sha1,
+ }, session_for(user)
+ assert_response :success
+ assert_select 'tr td a', 'COPYING'
+ assert_select 'tr td', '625 bytes'
+ assert_select 'tr td a', 'apps'
+ assert_select 'tr td a', 'workbench'
+ assert_select 'tr td a', 'Gemfile'
+ assert_select 'tr td', '33.7 KiB'
+ end
+
+ test "show commit to #{user}" do
+ reset_api_fixtures_after_test false
+ sha1, commit, _ = stub_repo_content
+ get :show_commit, {
+ id: api_fixture('repositories')['foo']['uuid'],
+ commit: sha1,
+ }, session_for(user)
+ assert_response :success
+ assert_select 'pre', h(commit)
+ end
+
+ test "show blob to #{user}" do
+ reset_api_fixtures_after_test false
+ sha1, _, filedata = stub_repo_content filename: 'COPYING'
+ get :show_blob, {
+ id: api_fixture('repositories')['foo']['uuid'],
+ commit: sha1,
+ path: 'COPYING',
+ }, session_for(user)
+ assert_response :success
+ assert_select 'pre', h(filedata)
+ end
+ end
+
+ ['', '/'].each do |path|
+ test "show tree with path '#{path}'" do
+ reset_api_fixtures_after_test false
+ sha1, _, _ = stub_repo_content filename: 'COPYING'
+ get :show_tree, {
+ id: api_fixture('repositories')['foo']['uuid'],
+ commit: sha1,
+ path: path,
+ }, session_for(:active)
+ assert_response :success
+ assert_select 'tr td', 'COPYING'
+ end
+ end
end
diff --git a/apps/workbench/test/helpers/repository_stub_helper.rb b/apps/workbench/test/helpers/repository_stub_helper.rb
new file mode 100644
index 0000000..b7d0573
--- /dev/null
+++ b/apps/workbench/test/helpers/repository_stub_helper.rb
@@ -0,0 +1,33 @@
+module RepositoryStubHelper
+ # Supply some fake git content.
+ def stub_repo_content opts={}
+ fakesha1 = opts[:sha1] || 'abcdefabcdefabcdefabcdefabcdefabcdefabcd'
+ fakefilename = opts[:filename] || 'COPYING'
+ fakefilesrc = File.expand_path('../../../../../'+fakefilename, __FILE__)
+ fakefile = File.read fakefilesrc
+ fakecommit = <<-EOS
+ commit abcdefabcdefabcdefabcdefabcdefabcdefabcd
+ Author: Fake R <fake at example.com>
+ Date: Wed Apr 1 11:59:59 2015 -0400
+
+ It's a fake commit.
+
+ EOS
+ Repository.any_instance.stubs(:ls_tree_lr).with(fakesha1).returns <<-EOS
+ 100644 blob eec475862e6ec2a87554e0fca90697e87f441bf5 226 .gitignore
+ 100644 blob acbd7523ed49f01217874965aa3180cccec89d61 625 COPYING
+ 100644 blob d645695673349e3947e8e5ae42332d0ac3164cd7 11358 LICENSE-2.0.txt
+ 100644 blob c7a36c355b4a2b94dfab45c9748330022a788c91 622 README
+ 100644 blob dba13ed2ddf783ee8118c6a581dbf75305f816a3 34520 agpl-3.0.txt
+ 100644 blob 9bef02bbfda670595750fd99a4461005ce5b8f12 695 apps/workbench/.gitignore
+ 100644 blob b51f674d90f68bfb50d9304068f915e42b04aea4 2249 apps/workbench/Gemfile
+ 100644 blob b51f674d90f68bfb50d9304068f915e42b04aea4 2249 apps/workbench/Gemfile
+ 100755 blob cdd5ebaff27781f93ab85e484410c0ce9e97770f 1012 crunch_scripts/hash
+ EOS
+ Repository.any_instance.
+ stubs(:cat_file).with(fakesha1, fakefilename).returns fakefile
+ Repository.any_instance.
+ stubs(:show).with(fakesha1).returns fakecommit
+ return fakesha1, fakecommit, fakefile
+ end
+end
diff --git a/apps/workbench/test/integration/repositories_browse_test.rb b/apps/workbench/test/integration/repositories_browse_test.rb
new file mode 100644
index 0000000..147bf46
--- /dev/null
+++ b/apps/workbench/test/integration/repositories_browse_test.rb
@@ -0,0 +1,37 @@
+require 'integration_helper'
+require 'helpers/repository_stub_helper'
+require 'helpers/share_object_helper'
+
+class RepositoriesTest < ActionDispatch::IntegrationTest
+ include RepositoryStubHelper
+ include ShareObjectHelper
+
+ reset_api_fixtures :after_each_test, false
+
+ setup do
+ need_javascript
+ end
+
+ test "browse repository from jobs#show" do
+ sha1 = api_fixture('jobs')['running']['script_version']
+ _, fakecommit, fakefile =
+ stub_repo_content sha1: sha1, filename: 'crunch_scripts/hash'
+ show_object_using 'active', 'jobs', 'running', sha1
+ click_on api_fixture('jobs')['running']['script']
+ assert_text fakefile
+ click_on 'crunch_scripts'
+ assert_selector 'td a', text: 'hash'
+ click_on 'foo'
+ assert_selector 'td a', text: 'crunch_scripts'
+ click_on sha1
+ assert_text fakecommit
+
+ show_object_using 'active', 'jobs', 'running', sha1
+ click_on 'active/foo'
+ assert_selector 'td a', text: 'crunch_scripts'
+
+ show_object_using 'active', 'jobs', 'running', sha1
+ click_on sha1
+ assert_text fakecommit
+ end
+end
diff --git a/apps/workbench/test/test_helper.rb b/apps/workbench/test/test_helper.rb
index fdde55d..4ab162c 100644
--- a/apps/workbench/test/test_helper.rb
+++ b/apps/workbench/test/test_helper.rb
@@ -270,12 +270,17 @@ class ActiveSupport::TestCase
end
def after_teardown
- if self.class.want_reset_api_fixtures[:after_each_test]
+ if self.class.want_reset_api_fixtures[:after_each_test] and
+ @want_reset_api_fixtures != false
self.class.reset_api_fixtures_now
end
super
end
+ def reset_api_fixtures_after_test t=true
+ @want_reset_api_fixtures = t
+ end
+
protected
def self.reset_api_fixtures_now
# Never try to reset fixtures when we're just using test
diff --git a/services/api/test/fixtures/jobs.yml b/services/api/test/fixtures/jobs.yml
index c662062..23b32ef 100644
--- a/services/api/test/fixtures/jobs.yml
+++ b/services/api/test/fixtures/jobs.yml
@@ -7,6 +7,8 @@ running:
created_at: <%= 3.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: ~
+ script: hash
+ repository: active/foo
script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
running: true
success: ~
@@ -31,6 +33,8 @@ running_cancelled:
created_at: <%= 4.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: ~
+ script: hash
+ repository: active/foo
script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
running: true
success: ~
@@ -56,6 +60,8 @@ uses_nonexistent_script_version:
created_at: <%= 5.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: <%= 2.minute.ago.to_s(:db) %>
+ script: hash
+ repository: active/foo
running: false
success: true
output: d41d8cd98f00b204e9800998ecf8427e+0
commit b8f941d086fdcc7d882a60c59090e2827fe1adec
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Mar 31 10:05:47 2015 -0400
5416: Fix comment.
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index 40c0bfe..8aeacf4 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -203,5 +203,5 @@ common:
# in the directory where your API server is running.
anonymous_user_token: false
- # Include Accept-Encoding header when making API requests
+ # Enable response payload compression in Arvados API requests.
include_accept_encoding_header_in_api_requests: true
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list