[ARVADOS] updated: 57d5f07da6dad0c48fa71788130718e9933a4551
git at public.curoverse.com
git at public.curoverse.com
Sat May 2 02:40:12 EDT 2015
Summary of changes:
apps/workbench/Gemfile | 3 +-
apps/workbench/Gemfile.lock | 5 +-
.../app/assets/javascripts/infinite_scroll.js | 34 +++
.../app/controllers/collections_controller.rb | 4 +-
apps/workbench/app/controllers/jobs_controller.rb | 2 -
.../controllers/pipeline_instances_controller.rb | 4 +-
.../app/controllers/projects_controller.rb | 2 +-
apps/workbench/app/helpers/application_helper.rb | 6 +-
apps/workbench/app/helpers/jobs_helper.rb | 22 --
apps/workbench/app/models/pipeline_instance.rb | 52 ++++
.../app/views/application/_choose.html.erb | 4 +
.../app/views/application/_content.html.erb | 12 +-
.../app/views/application/_show_sharing.html.erb | 1 +
.../app/views/collections/hash_matches.html.erb | 8 +-
.../app/views/layouts/application.html.erb | 6 +-
apps/workbench/app/views/layouts/body.html.erb | 11 +-
.../views/pipeline_instances/_show_log.html.erb | 56 +++-
apps/workbench/config/database.yml | 40 +--
.../test/controllers/projects_controller_test.rb | 27 ++
apps/workbench/test/helpers/share_object_helper.rb | 1 +
.../test/integration/anonymous_access_test.rb | 33 +--
.../test/integration/application_layout_test.rb | 4 +-
.../workbench/test/integration/collections_test.rb | 20 +-
apps/workbench/test/integration/errors_test.rb | 4 +-
.../test/integration/pipeline_instances_test.rb | 41 +++
.../test/integration/user_manage_account_test.rb | 6 +-
.../test/integration/user_profile_test.rb | 4 +-
apps/workbench/test/unit/pipeline_instance_test.rb | 66 ++++-
crunch_scripts/run-command | 2 +-
.../install-workbench-app.html.textile.liquid | 2 +-
sdk/go/arvadosclient/arvadosclient.go | 64 +++--
sdk/go/arvadosclient/arvadosclient_test.go | 14 +-
sdk/python/arvados/arvfile.py | 15 +-
sdk/python/arvados/errors.py | 9 +-
sdk/python/arvados/keep.py | 309 +++++++++++++++------
sdk/python/arvados/retry.py | 19 +-
sdk/python/setup.py | 33 +--
sdk/python/tests/arvados_testutil.py | 106 ++++---
sdk/python/tests/keepstub.py | 95 +++++++
sdk/python/tests/test_arvfile.py | 19 +-
sdk/python/tests/test_collections.py | 21 +-
sdk/python/tests/test_errors.py | 4 +-
sdk/python/tests/test_keep_client.py | 221 ++++++++++-----
sdk/python/tests/test_retry.py | 5 +-
sdk/python/tests/test_stream.py | 16 +-
services/api/Rakefile | 21 ++
.../api/app/controllers/application_controller.rb | 37 ++-
.../arvados/v1/collections_controller.rb | 4 +
.../app/controllers/arvados/v1/jobs_controller.rb | 27 +-
services/api/app/models/arvados_model.rb | 7 +
services/api/config/application.default.yml | 15 +
services/api/lib/tasks/delete_old_job_logs.rake | 17 ++
services/api/lib/tasks/test_tasks.rake | 6 +
services/api/test/fixtures/groups.yml | 26 ++
services/api/test/fixtures/jobs.yml | 53 +++-
services/api/test/fixtures/links.yml | 30 ++
services/api/test/fixtures/logs.yml | 38 +++
services/api/test/fixtures/pipeline_instances.yml | 42 ++-
services/api/test/fixtures/pipeline_templates.yml | 40 +++
services/api/test/fixtures/repositories.yml | 7 +
.../arvados/v1/collections_controller_test.rb | 39 +++
.../arvados/v1/job_reuse_controller_test.rb | 108 +++++--
.../api/test/tasks/delete_old_job_logs_test.rb | 50 ++++
services/api/test/test.git.tar | Bin 194560 -> 256000 bytes
services/api/test/unit/collection_test.rb | 7 +
services/api/test/unit/commit_test.rb | 60 ++++
services/api/test/unit/job_test.rb | 10 +
services/keepstore/handlers.go | 52 ++--
services/keepstore/keepstore.go | 4 +-
69 files changed, 1668 insertions(+), 464 deletions(-)
delete mode 100644 apps/workbench/app/helpers/jobs_helper.rb
create mode 100644 sdk/python/tests/keepstub.py
create mode 100644 services/api/lib/tasks/delete_old_job_logs.rake
create mode 100644 services/api/lib/tasks/test_tasks.rake
create mode 100644 services/api/test/tasks/delete_old_job_logs_test.rb
discards 00121a241b60d5aca578e98c379e35b92e759c2b (commit)
discards 0f1d3aca63546ecbb4ad1d05ec7955258ea3bd44 (commit)
discards c0a80b0a49ab007440cb314d36055b5a23c606ab (commit)
discards 9019ae81f586fe10e6687710070c34e6ba00aecd (commit)
via 57d5f07da6dad0c48fa71788130718e9933a4551 (commit)
via 33235ab194d91a65df2774a3242edaa8556eb369 (commit)
via c311c44755d92a270699faee90389bb928833c4c (commit)
via 42b4e23e61514c6578791ddd23de601c689d8512 (commit)
via e8f91c87ef238ddf6bd96914a196d2d90825e8ee (commit)
via 3f51fa4899fe43c29e1bf49d7911a40eb41a55e8 (commit)
via 91c467665ad493108b69c660d5424c77cce5668f (commit)
via 6a7271ad8b9c145c6fb6bbc9e0914f32bcaa225d (commit)
via ee8d5b171e2c30ddc3fe829ba4388778e6a78b27 (commit)
via f81a5a1adb403f9c81159239e92880843dca3c93 (commit)
via db195a5479485501eb59ce5e914eddc64f72a9f5 (commit)
via 7990a7778f70c243b50ea878787ea83689f5b07e (commit)
via dffb8b5eccb3af901512effe562a8b8944b14b89 (commit)
via c42ccd1f88e75b7910623581fbdf8d88c42b55dd (commit)
via 02676af9f5a2580710e9ae062e7028f661740c3a (commit)
via e94da792a8fc562bc35e63b6ff5b2f87822d2755 (commit)
via 4570521e9d639806b1f25c93822d5cf0dbfe42d0 (commit)
via f94d59e86fe1e65c18cf905fa9ea1fcabfd1b1b5 (commit)
via 3da2281b33aa42a6885ee12d930843c3d43a1ffb (commit)
via ec359b66ac1619511757ee78e020536f2ae13581 (commit)
via 624c5c8d13d6c2e21a80379928387944bedae2a3 (commit)
via a4211acb465bd42869bf2a2f9fad6ff2c5e518e0 (commit)
via 6c9dde5e6089efc8a87022ab9771da96932516a7 (commit)
via 8af4c5ffe2d0bf7f009f8821945aa100ddf02279 (commit)
via acc7d4a40ae78e866c983624ec36b7bae3b918a9 (commit)
via ea7c7efb7d2d7429677f6d95eb92128d7574d6a5 (commit)
via 213ace8be1b2279e89abcaf58de004eb8b58632c (commit)
via e24940267012211db40ca0adcedfc33650314b70 (commit)
via 61d399a57fd1e867cc243a0dab0fad73d9689734 (commit)
via e01762a0adcdee209f305cfc2ae902ab4eb94899 (commit)
via 20b988d5b901f459f95043c3702f1f9b104f3fbf (commit)
via 7ab494e42b35b3769a326a16b7e90f1d20147ced (commit)
via 0191b58223f99476cc8553f3ad7c726a4d8b37de (commit)
via de5f63b3abc80783ebe886078c5e4bcfa19ed1da (commit)
via 48c6eca6aa6af517f67d39545b714946676cd2b5 (commit)
via ec0c9f0d12560296d4b678c8ad33e1b82221e626 (commit)
via 19db486cda40c892afb1111bb3d35d482ab1fbb2 (commit)
via b6c7ee612c09a110b6e2a87a12eebc5b81a7a2e8 (commit)
via 040fdf6da00318b5b8e8e83eceeec783680ecb1c (commit)
via 8464bb48ba769c8bf0984c8b1493ca43fd902891 (commit)
via 5b1602a521d3f6689f5988cdc2b25b74ff9fc1e1 (commit)
via 03d89302f14a4a1c448b32d62679bab14e23fa23 (commit)
via 68dfd41d1d5038f7e0a139779ef6ce1c4a7a8351 (commit)
via 5b0d540ad315ad390831ad58f52b02e86686524f (commit)
via 4b17e78749b44772e898619e11aef663810ac6ed (commit)
via d054beda9bc86bc6cd33a127f4d9e40d04f1620d (commit)
via 9bfd0d772abcd643e92fed127112e0ed91c00e65 (commit)
via 11a007fd7bbc3b35b6efd62c84d9ca5a8f45aecb (commit)
via 97403e08475b328115373a2c6a23e82116199aad (commit)
via 45b556d1a204f694b69ca26d115cd6786ca585f3 (commit)
via 123ef960b82540b98c8b68c47ff89bf85da3899c (commit)
via 81fb9f6e13d6e48e23165319ff8f2f9cab4deb3f (commit)
via b1db376a4532ba3484929b3214e59056877d1695 (commit)
via 816683553b8353f18c92f32a6ee9ea1868630067 (commit)
via 41068df20f2a9fe9dbb3e29e55af0c82832359ad (commit)
via f6f6ed672a240c9b91e78591b4f67a5ed4250758 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (00121a241b60d5aca578e98c379e35b92e759c2b)
\
N -- N -- N (57d5f07da6dad0c48fa71788130718e9933a4551)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 57d5f07da6dad0c48fa71788130718e9933a4551
Author: Tom Clegg <tom at curoverse.com>
Date: Sat May 2 02:34:03 2015 -0400
5416: Remove "disable repo browsing with git2" stuff.
git2 is no longer unreliable, now that we know the silent failures
meant "git credential helpers must consume their stdin".
diff --git a/apps/workbench/app/controllers/repositories_controller.rb b/apps/workbench/app/controllers/repositories_controller.rb
index 89dd96b..c5b3501 100644
--- a/apps/workbench/app/controllers/repositories_controller.rb
+++ b/apps/workbench/app/controllers/repositories_controller.rb
@@ -1,8 +1,5 @@
class RepositoriesController < ApplicationController
before_filter :set_share_links, if: -> { defined? @object }
- if Repository.disable_repository_browsing?
- before_filter :render_browsing_disabled, only: [:show_tree, :show_blob, :show_commit]
- end
def index_pane_list
%w(recent help)
@@ -35,10 +32,4 @@ class RepositoriesController < ApplicationController
def show_commit
@commit = params[:commit]
end
-
- protected
-
- def render_browsing_disabled
- render_not_found ActionController::RoutingError.new("Repository browsing features disabled")
- end
end
diff --git a/apps/workbench/app/models/repository.rb b/apps/workbench/app/models/repository.rb
index 6aa6bb7..1caab89 100644
--- a/apps/workbench/app/models/repository.rb
+++ b/apps/workbench/app/models/repository.rb
@@ -48,15 +48,6 @@ class Repository < ArvadosBase
subtree
end
- # git 2.1.4 does not use credential helpers reliably, see #5416
- def self.disable_repository_browsing?
- return false if Rails.configuration.use_git2_despite_bug_risk
- if @buggy_git_version.nil?
- @buggy_git_version = /git version 2/ =~ `git version`
- end
- @buggy_git_version
- end
-
# http_fetch_url returns the first http:// or https:// url (if any)
# in the api response's clone_urls attribute.
def http_fetch_url
diff --git a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
index 80210c4..2ab8da1 100644
--- a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
@@ -99,8 +99,7 @@
<% # link to repo tree/file only if the repo is readable
# and the commit is a sha1...
repo =
- (not Repository.disable_repository_browsing? and
- /^[0-9a-f]{40}$/ =~ current_component[:script_version] and
+ (/^[0-9a-f]{40}$/ =~ current_component[:script_version] and
Repository.where(name: current_component[:repository]).first)
# ...and the api server provides an http:// or https:// url
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index e28f76a..4061ee8 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -211,9 +211,3 @@ common:
# Enable response payload compression in Arvados API requests.
include_accept_encoding_header_in_api_requests: true
-
- # Enable repository browsing even if git2 is installed. Repository
- # browsing requires credential helpers, which do not work reliably
- # as of git version 2.1.4. If you have git version 2.* and you want
- # to use it anyway, change this to true.
- use_git2_despite_bug_risk: false
diff --git a/apps/workbench/test/controllers/repositories_controller_test.rb b/apps/workbench/test/controllers/repositories_controller_test.rb
index 852a602..25bf557 100644
--- a/apps/workbench/test/controllers/repositories_controller_test.rb
+++ b/apps/workbench/test/controllers/repositories_controller_test.rb
@@ -69,7 +69,6 @@ class RepositoriesControllerTest < ActionController::TestCase
[:active, :spectator].each do |user|
test "show tree to #{user}" do
- skip "git2 is unreliable" if Repository.disable_repository_browsing?
reset_api_fixtures_after_test false
sha1, _, _ = stub_repo_content
get :show_tree, {
@@ -86,7 +85,6 @@ class RepositoriesControllerTest < ActionController::TestCase
end
test "show commit to #{user}" do
- skip "git2 is unreliable" if Repository.disable_repository_browsing?
reset_api_fixtures_after_test false
sha1, commit, _ = stub_repo_content
get :show_commit, {
@@ -98,7 +96,6 @@ class RepositoriesControllerTest < ActionController::TestCase
end
test "show blob to #{user}" do
- skip "git2 is unreliable" if Repository.disable_repository_browsing?
reset_api_fixtures_after_test false
sha1, _, filedata = stub_repo_content filename: 'COPYING'
get :show_blob, {
@@ -113,7 +110,6 @@ class RepositoriesControllerTest < ActionController::TestCase
['', '/'].each do |path|
test "show tree with path '#{path}'" do
- skip "git2 is unreliable" if Repository.disable_repository_browsing?
reset_api_fixtures_after_test false
sha1, _, _ = stub_repo_content filename: 'COPYING'
get :show_tree, {
diff --git a/apps/workbench/test/integration/repositories_browse_test.rb b/apps/workbench/test/integration/repositories_browse_test.rb
index d936877..a6a85b5 100644
--- a/apps/workbench/test/integration/repositories_browse_test.rb
+++ b/apps/workbench/test/integration/repositories_browse_test.rb
@@ -13,7 +13,6 @@ class RepositoriesTest < ActionDispatch::IntegrationTest
end
test "browse repository from jobs#show" do
- skip "git2 is unreliable" if Repository.disable_repository_browsing?
sha1 = api_fixture('jobs')['running']['script_version']
_, fakecommit, fakefile =
stub_repo_content sha1: sha1, filename: 'crunch_scripts/hash'
@@ -37,7 +36,6 @@ class RepositoriesTest < ActionDispatch::IntegrationTest
end
test "browse using arv-git-http" do
- skip "git2 is unreliable" if Repository.disable_repository_browsing?
repo = api_fixture('repositories')['foo']
portfile =
File.expand_path('../../../../../tmp/arv-git-httpd-ssl.port', __FILE__)
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index a86de69..e3d574e 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -3,6 +3,7 @@
--
SET statement_timeout = 0;
+SET lock_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SET check_function_bodies = false;
commit 33235ab194d91a65df2774a3242edaa8556eb369
Author: Tom Clegg <tom at curoverse.com>
Date: Sat May 2 02:16:19 2015 -0400
5416: Use git credential helpers for arv-git-httpd tests. Fix workbench helper.
The sporadic "git exited 128" errors -- very common in git2 and rare
in git1 -- seem to have been caused by git getting SIGPIPE when the
credential helper exited without consuming stdin. The solution is for
the credential helper to discard its standard input before exiting.
diff --git a/apps/workbench/app/models/repository.rb b/apps/workbench/app/models/repository.rb
index b9b9ce3..6aa6bb7 100644
--- a/apps/workbench/app/models/repository.rb
+++ b/apps/workbench/app/models/repository.rb
@@ -96,7 +96,7 @@ class Repository < ArvadosBase
"credential.#{http_fetch_url}.username", 'none'],
['git', '--git-dir', @workdir, 'config', '--local',
"credential.#{http_fetch_url}.helper",
- '!token(){ echo password="$ARVADOS_API_TOKEN"; }; token'],
+ '!cred(){ cat >/dev/null; if [ "$1" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred'],
['git', '--git-dir', @workdir, 'config', '--local',
'http.sslVerify',
Rails.configuration.arvados_insecure_https ? 'false' : 'true'],
diff --git a/services/arv-git-httpd/server_test.go b/services/arv-git-httpd/server_test.go
index 5bf861b..e5ddc29 100644
--- a/services/arv-git-httpd/server_test.go
+++ b/services/arv-git-httpd/server_test.go
@@ -91,6 +91,17 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
_, err = exec.Command("sh", "-c", "cd "+s.tmpWorkdir+" && echo work >work && git add work && git -c user.name=Foo -c user.email=Foo commit -am 'workdir: test'").CombinedOutput()
c.Assert(err, check.Equals, nil)
+ _, err = exec.Command("git", "config",
+ "--file", s.tmpWorkdir+"/.git/config",
+ "credential.http://"+s.testServer.Addr+"/.helper",
+ "!cred(){ cat >/dev/null; if [ \"$1\" = get ]; then echo password=$ARVADOS_API_TOKEN; fi; };cred").Output()
+ c.Assert(err, check.Equals, nil)
+ _, err = exec.Command("git", "config",
+ "--file", s.tmpWorkdir+"/.git/config",
+ "credential.http://"+s.testServer.Addr+"/.username",
+ "none").Output()
+ c.Assert(err, check.Equals, nil)
+
theConfig = &config{
Addr: ":",
GitCommand: "/usr/bin/git",
@@ -127,9 +138,10 @@ func (s *IntegrationSuite) runGit(c *check.C, token, gitCmd, repo string, args .
os.Chdir(s.tmpWorkdir)
gitargs := append([]string{
- gitCmd, "http://none:" + token + "@" + s.testServer.Addr + "/" + repo,
+ gitCmd, "http://" + s.testServer.Addr + "/" + repo,
}, args...)
cmd := exec.Command("git", gitargs...)
+ cmd.Env = append(os.Environ(), "ARVADOS_API_TOKEN="+token)
w, err := cmd.StdinPipe()
c.Assert(err, check.Equals, nil)
w.Close()
commit c311c44755d92a270699faee90389bb928833c4c
Author: Tom Clegg <tom at curoverse.com>
Date: Sat May 2 02:09:54 2015 -0400
5416: gofmt fixes. Fix some logs to print as strings, not byte arrays.
diff --git a/services/arv-git-httpd/basic_auth_test.go b/services/arv-git-httpd/basic_auth_test.go
index 6bc88cb..2bd84dc 100644
--- a/services/arv-git-httpd/basic_auth_test.go
+++ b/services/arv-git-httpd/basic_auth_test.go
@@ -14,15 +14,15 @@ type basicAuthTestCase struct {
func TestBasicAuth(t *testing.T) {
tests := []basicAuthTestCase{
- basicAuthTestCase{"Basic Zm9vOmJhcg==", "foo", "bar", true},
- basicAuthTestCase{"Bogus Zm9vOmJhcg==", "", "", false},
- basicAuthTestCase{"Zm9vOmJhcg==", "", "", false},
- basicAuthTestCase{"Basic", "", "", false},
- basicAuthTestCase{"", "", "", false},
+ {"Basic Zm9vOmJhcg==", "foo", "bar", true},
+ {"Bogus Zm9vOmJhcg==", "", "", false},
+ {"Zm9vOmJhcg==", "", "", false},
+ {"Basic", "", "", false},
+ {"", "", "", false},
}
for _, test := range tests {
if u, p, ok := BasicAuth(&http.Request{Header: map[string][]string{
- "Authorization": []string{test.hdr},
+ "Authorization": {test.hdr},
}}); u != test.user || p != test.pass || ok != test.ok {
t.Error("got:", u, p, ok, "expected:", test.user, test.pass, test.ok, "from:", test.hdr)
}
diff --git a/services/arv-git-httpd/server_test.go b/services/arv-git-httpd/server_test.go
index ca5c5f4..5bf861b 100644
--- a/services/arv-git-httpd/server_test.go
+++ b/services/arv-git-httpd/server_test.go
@@ -149,10 +149,10 @@ func (s *IntegrationSuite) runGit(c *check.C, token, gitCmd, repo string, args .
// Make a bare arvados repo at {tmpRepoRoot}/arvados.git
func (s *IntegrationSuite) makeArvadosRepo(c *check.C) {
msg, err := exec.Command("git", "init", "--bare", s.tmpRepoRoot+"/zzzzz-s0uqq-arvadosrepo0123.git").CombinedOutput()
- c.Log(msg)
+ c.Log(string(msg))
c.Assert(err, check.Equals, nil)
msg, err = exec.Command("git", "--git-dir", s.tmpRepoRoot+"/zzzzz-s0uqq-arvadosrepo0123.git", "fetch", "../../.git", "HEAD:master").CombinedOutput()
- c.Log(msg)
+ c.Log(string(msg))
c.Assert(err, check.Equals, nil)
}
commit 42b4e23e61514c6578791ddd23de601c689d8512
Author: Tom Clegg <tom at curoverse.com>
Date: Fri May 1 19:47:09 2015 -0400
5416: Add realm="git" to WWW-Authenticate header, cf. rfc2617
diff --git a/services/arv-git-httpd/auth_handler.go b/services/arv-git-httpd/auth_handler.go
index df2d8d6..6313d50 100644
--- a/services/arv-git-httpd/auth_handler.go
+++ b/services/arv-git-httpd/auth_handler.go
@@ -60,7 +60,7 @@ func (h *authHandler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
username, password, ok := BasicAuth(r)
if !ok || username == "" || password == "" {
statusCode, statusText = http.StatusUnauthorized, "no credentials provided"
- w.Header().Add("WWW-Authenticate", "basic")
+ w.Header().Add("WWW-Authenticate", "Basic realm=\"git\"")
return
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list