[ARVADOS] created: dbbc879dec9ce61a64176925b8aeb1a38240817c
git at public.curoverse.com
git at public.curoverse.com
Wed Apr 1 18:27:37 EDT 2015
at dbbc879dec9ce61a64176925b8aeb1a38240817c (commit)
commit dbbc879dec9ce61a64176925b8aeb1a38240817c
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 1 18:28:19 2015 -0400
5416: Browse git repository contents in workbench.
diff --git a/apps/workbench/app/controllers/repositories_controller.rb b/apps/workbench/app/controllers/repositories_controller.rb
index d32c92a..c5b3501 100644
--- a/apps/workbench/app/controllers/repositories_controller.rb
+++ b/apps/workbench/app/controllers/repositories_controller.rb
@@ -16,4 +16,20 @@ class RepositoriesController < ApplicationController
panes.delete('Attributes') if !current_user.is_admin
panes
end
+
+ def show_tree
+ @commit = params[:commit]
+ @path = params[:path] || ''
+ @subtree = @object.ls_subtree @commit, @path.chomp('/')
+ end
+
+ def show_blob
+ @commit = params[:commit]
+ @path = params[:path]
+ @blobdata = @object.cat_file @commit, @path
+ end
+
+ def show_commit
+ @commit = params[:commit]
+ end
end
diff --git a/apps/workbench/app/models/repository.rb b/apps/workbench/app/models/repository.rb
index b062dda..4d0430b 100644
--- a/apps/workbench/app/models/repository.rb
+++ b/apps/workbench/app/models/repository.rb
@@ -12,4 +12,105 @@ class Repository < ArvadosBase
[]
end
end
+
+ def show commit_sha1
+ refresh
+ run_git 'show', commit_sha1
+ end
+
+ def cat_file commit_sha1, path
+ refresh
+ run_git 'cat-file', 'blob', commit_sha1 + ':' + path
+ end
+
+ def ls_tree_lr commit_sha1
+ refresh
+ run_git 'ls-tree', '-l', '-r', commit_sha1
+ end
+
+ # subtree returns a list of files under the given path at the
+ # specified commit. Results are returned as an array of file nodes,
+ # where each file node is an array [file mode, blob sha1, file size
+ # in bytes, path relative to the given directory]. If the path is
+ # not found, [] is returned.
+ def ls_subtree commit, path
+ path = path.chomp '/'
+ subtree = []
+ ls_tree_lr(commit).each_line do |line|
+ mode, type, sha1, size, filepath = line.split
+ next if type != 'blob'
+ if filepath[0,path.length] == path and
+ (path == '' or filepath[path.length] == '/')
+ subtree << [mode.to_i(8), sha1, size.to_i,
+ filepath[path.length,filepath.length]]
+ end
+ end
+ subtree
+ end
+
+ protected
+
+ # refresh fetches the latest repository content into the local
+ # cache. It is a no-op if it has already been run on this object:
+ # this (pretty much) avoids doing more than one remote git operation
+ # per Workbench request.
+ def refresh
+ run_git 'fetch', http_fetch_url, '+*:*' unless @fresh
+ @fresh = true
+ end
+
+ # http_fetch_url returns an http:// or https:// fetch-url which can
+ # accept arvados API token authentication. The API server currently
+ # advertises SSH fetch-urls, which work for users with SSH keys but
+ # are useless for fetching repository content into workbench itself.
+ def http_fetch_url
+ "https://git.#{uuid[0,5]}.arvadosapi.com/#{name}.git"
+ end
+
+ # run_git sets up the ARVADOS_API_TOKEN environment variable,
+ # creates a local git directory for this repository if necessary,
+ # executes "git --git-dir localgitdir {args to run_git}", and
+ # returns the output. It raises GitCommandError if git exits
+ # non-zero.
+ def run_git *cmd
+ if not @workdir
+ workdir = File.expand_path uuid+'.git', Rails.configuration.repository_cache
+ if not File.exists? workdir
+ FileUtils.mkdir_p Rails.configuration.repository_cache
+ [['git', 'init', '--bare', workdir],
+ ].each do |cmd|
+ system *cmd
+ raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0
+ end
+ end
+ @workdir = workdir
+ end
+ [['git', '--git-dir', @workdir, 'config', '--local',
+ "credential.#{http_fetch_url}.username", 'none'],
+ ['git', '--git-dir', @workdir, 'config', '--local',
+ "credential.#{http_fetch_url}.helper",
+ '!token(){ echo password="$ARVADOS_API_TOKEN"; }; token'],
+ ['git', '--git-dir', @workdir, 'config', '--local',
+ 'http.sslVerify',
+ Rails.configuration.arvados_insecure_https ? 'false' : 'true'],
+ ].each do |cmd|
+ system *cmd
+ raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0
+ end
+ env = {}.
+ merge(ENV).
+ merge('ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token])
+ cmd = ['git', '--git-dir', @workdir] + cmd
+ io = IO.popen(env, cmd, err: [:child, :out])
+ output = io.read
+ io.close
+ # "If [io] is opened by IO.popen, close sets $?." --ruby 2.2.1 docs
+ unless $?.exitstatus == 0
+ raise GitCommandError.new(cmd.inspect + "\n" + output)
+ end
+ output
+ end
+
+ class GitCommandError < StandardError
+ end
end
diff --git a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
index 018d49f..ab318e1 100644
--- a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
@@ -96,6 +96,12 @@
<div class="row">
<div class="col-md-6">
<table>
+ <% # link to repo tree/file only if the repo is readable
+ # and the commit is a sha1
+ repo =
+ (/^[0-9a-f]{40}$/ =~ current_component[:script_version] and
+ Repository.where(name: current_component[:repository]).first)
+ %>
<% [:script, :repository, :script_version, :supplied_script_version, :nondeterministic].each do |k| %>
<tr>
<td style="padding-right: 1em">
@@ -104,6 +110,12 @@
<td>
<% if current_component[k].nil? %>
(none)
+ <% elsif k == :repository %>
+ <%= link_to_if repo, current_component[k], show_repository_tree_path(id: repo.uuid, commit: current_component[:script_version], path: '/') %>
+ <% elsif k == :script %>
+ <%= link_to_if repo, current_component[k], show_repository_blob_path(id: repo.uuid, commit: current_component[:script_version], path: 'crunch_scripts/'+current_component[:script]) %>
+ <% elsif k == :script_version %>
+ <%= link_to_if repo, current_component[k], show_repository_commit_path(id: repo.uuid, commit: current_component[:script_version]) %>
<% else %>
<%= current_component[k] %>
<% end %>
diff --git a/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb b/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb
new file mode 100644
index 0000000..736c187
--- /dev/null
+++ b/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb
@@ -0,0 +1,13 @@
+<div class="pull-right">
+ <span class="deemphasize">Browsing <%= @object.name %> repository at commit</span>
+ <%= link_to(@commit, show_repository_commit_path(id: @object.uuid, commit: @commit), title: 'show commit message') %>
+</div>
+<p>
+ <%= link_to(@object.name, show_repository_tree_path(id: @object.uuid, commit: @commit, path: '/'), title: 'show root directory of source tree') %>
+ <% parents = ''
+ (@path || '').split('/').each do |pathpart|
+ parents = parents + pathpart + '/'
+ %>
+ / <%= link_to pathpart, show_repository_tree_path(id: @object.uuid, commit: @commit, path: parents) %>
+ <% end %>
+</p>
diff --git a/apps/workbench/app/views/repositories/show_blob.html.erb b/apps/workbench/app/views/repositories/show_blob.html.erb
new file mode 100644
index 0000000..acc34d1
--- /dev/null
+++ b/apps/workbench/app/views/repositories/show_blob.html.erb
@@ -0,0 +1,13 @@
+<%= render partial: 'repository_breadcrumbs' %>
+
+<% if not @blobdata.valid_encoding? %>
+ <div class="alert alert-warning">
+ <p>
+ This file has an invalid text encoding, so it can't be shown
+ here. (This probably just means it's a binary file, not a text
+ file.)
+ </p>
+ </div>
+<% else %>
+ <pre><%= @blobdata %></pre>
+<% end %>
diff --git a/apps/workbench/app/views/repositories/show_commit.html.erb b/apps/workbench/app/views/repositories/show_commit.html.erb
new file mode 100644
index 0000000..3690be6
--- /dev/null
+++ b/apps/workbench/app/views/repositories/show_commit.html.erb
@@ -0,0 +1,3 @@
+<%= render partial: 'repository_breadcrumbs' %>
+
+<pre><%= @object.show @commit %></pre>
diff --git a/apps/workbench/app/views/repositories/show_tree.html.erb b/apps/workbench/app/views/repositories/show_tree.html.erb
new file mode 100644
index 0000000..4e2fcec
--- /dev/null
+++ b/apps/workbench/app/views/repositories/show_tree.html.erb
@@ -0,0 +1,40 @@
+<%= render partial: 'repository_breadcrumbs' %>
+
+<table class="table table-condensed table-hover">
+ <thead>
+ <tr>
+ <th>File</th>
+ <th class="data-size">Size</th>
+ </tr>
+ </thead>
+ <tbody>
+ <% @subtree.each do |mode, sha1, size, subpath| %>
+ <tr>
+ <td>
+ <span style="opacity: 0.6">
+ <% pathparts = subpath.sub(/^\//, '').split('/')
+ basename = pathparts.pop
+ parents = @path
+ pathparts.each do |pathpart| %>
+ <% parents = parents + '/' + pathpart %>
+ <%= link_to pathpart, url_for(path: parents) %>
+ /
+ <% end %>
+ </span>
+ <%= link_to basename, url_for(action: :show_blob, path: parents + '/' + basename) %>
+ </td>
+ <td class="data-size">
+ <%= human_readable_bytes_html(size) %>
+ </td>
+ </tr>
+ <% end %>
+ <% if @subtree.empty? %>
+ <tr>
+ <td>
+ No files found.
+ </td>
+ </tr>
+ <% end %>
+ </tbody>
+ <tfoot></tfoot>
+</table>
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index 8fa31e7..5b2391f 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -138,8 +138,14 @@ common:
default_openid_prefix: https://www.google.com/accounts/o8/id
send_user_setup_notification_email: true
- # Set user_profile_form_fields to enable and configure the user profile page.
- # Default is set to false. A commented setting with full description is provided below.
+ # Scratch directory used by the remote repository browsing
+ # feature. If it doesn't exist, it (and any missing parents) will be
+ # created using mkdir_p.
+ repository_cache: <%= File.expand_path 'tmp/git', Rails.root %>
+
+ # Set user_profile_form_fields to enable and configure the user
+ # profile page. Default is set to false. A commented example with
+ # full description is provided below.
user_profile_form_fields: false
# Below is a sample setting of user_profile_form_fields config parameter.
diff --git a/apps/workbench/config/routes.rb b/apps/workbench/config/routes.rb
index 7ed02e7..44d7ded 100644
--- a/apps/workbench/config/routes.rb
+++ b/apps/workbench/config/routes.rb
@@ -26,6 +26,11 @@ ArvadosWorkbench::Application.routes.draw do
resources :repositories do
post 'share_with', on: :member
end
+ # {format: false} prevents rails from treating "foo.png" as foo?format=png
+ get '/repositories/:id/tree/:commit' => 'repositories#show_tree'
+ get '/repositories/:id/tree/:commit/*path' => 'repositories#show_tree', as: :show_repository_tree, format: false
+ get '/repositories/:id/blob/:commit/*path' => 'repositories#show_blob', as: :show_repository_blob, format: false
+ get '/repositories/:id/commit/:commit' => 'repositories#show_commit', as: :show_repository_commit
match '/logout' => 'sessions#destroy', via: [:get, :post]
get '/logged_out' => 'sessions#index'
resources :users do
diff --git a/apps/workbench/test/controllers/repositories_controller_test.rb b/apps/workbench/test/controllers/repositories_controller_test.rb
index f95bb77..25bf557 100644
--- a/apps/workbench/test/controllers/repositories_controller_test.rb
+++ b/apps/workbench/test/controllers/repositories_controller_test.rb
@@ -1,7 +1,9 @@
require 'test_helper'
+require 'helpers/repository_stub_helper'
require 'helpers/share_object_helper'
class RepositoriesControllerTest < ActionController::TestCase
+ include RepositoryStubHelper
include ShareObjectHelper
[
@@ -62,4 +64,61 @@ class RepositoriesControllerTest < ActionController::TestCase
end
end
end
+
+ ### Browse repository content
+
+ [:active, :spectator].each do |user|
+ test "show tree to #{user}" do
+ reset_api_fixtures_after_test false
+ sha1, _, _ = stub_repo_content
+ get :show_tree, {
+ id: api_fixture('repositories')['foo']['uuid'],
+ commit: sha1,
+ }, session_for(user)
+ assert_response :success
+ assert_select 'tr td a', 'COPYING'
+ assert_select 'tr td', '625 bytes'
+ assert_select 'tr td a', 'apps'
+ assert_select 'tr td a', 'workbench'
+ assert_select 'tr td a', 'Gemfile'
+ assert_select 'tr td', '33.7 KiB'
+ end
+
+ test "show commit to #{user}" do
+ reset_api_fixtures_after_test false
+ sha1, commit, _ = stub_repo_content
+ get :show_commit, {
+ id: api_fixture('repositories')['foo']['uuid'],
+ commit: sha1,
+ }, session_for(user)
+ assert_response :success
+ assert_select 'pre', h(commit)
+ end
+
+ test "show blob to #{user}" do
+ reset_api_fixtures_after_test false
+ sha1, _, filedata = stub_repo_content filename: 'COPYING'
+ get :show_blob, {
+ id: api_fixture('repositories')['foo']['uuid'],
+ commit: sha1,
+ path: 'COPYING',
+ }, session_for(user)
+ assert_response :success
+ assert_select 'pre', h(filedata)
+ end
+ end
+
+ ['', '/'].each do |path|
+ test "show tree with path '#{path}'" do
+ reset_api_fixtures_after_test false
+ sha1, _, _ = stub_repo_content filename: 'COPYING'
+ get :show_tree, {
+ id: api_fixture('repositories')['foo']['uuid'],
+ commit: sha1,
+ path: path,
+ }, session_for(:active)
+ assert_response :success
+ assert_select 'tr td', 'COPYING'
+ end
+ end
end
diff --git a/apps/workbench/test/helpers/repository_stub_helper.rb b/apps/workbench/test/helpers/repository_stub_helper.rb
new file mode 100644
index 0000000..b7d0573
--- /dev/null
+++ b/apps/workbench/test/helpers/repository_stub_helper.rb
@@ -0,0 +1,33 @@
+module RepositoryStubHelper
+ # Supply some fake git content.
+ def stub_repo_content opts={}
+ fakesha1 = opts[:sha1] || 'abcdefabcdefabcdefabcdefabcdefabcdefabcd'
+ fakefilename = opts[:filename] || 'COPYING'
+ fakefilesrc = File.expand_path('../../../../../'+fakefilename, __FILE__)
+ fakefile = File.read fakefilesrc
+ fakecommit = <<-EOS
+ commit abcdefabcdefabcdefabcdefabcdefabcdefabcd
+ Author: Fake R <fake at example.com>
+ Date: Wed Apr 1 11:59:59 2015 -0400
+
+ It's a fake commit.
+
+ EOS
+ Repository.any_instance.stubs(:ls_tree_lr).with(fakesha1).returns <<-EOS
+ 100644 blob eec475862e6ec2a87554e0fca90697e87f441bf5 226 .gitignore
+ 100644 blob acbd7523ed49f01217874965aa3180cccec89d61 625 COPYING
+ 100644 blob d645695673349e3947e8e5ae42332d0ac3164cd7 11358 LICENSE-2.0.txt
+ 100644 blob c7a36c355b4a2b94dfab45c9748330022a788c91 622 README
+ 100644 blob dba13ed2ddf783ee8118c6a581dbf75305f816a3 34520 agpl-3.0.txt
+ 100644 blob 9bef02bbfda670595750fd99a4461005ce5b8f12 695 apps/workbench/.gitignore
+ 100644 blob b51f674d90f68bfb50d9304068f915e42b04aea4 2249 apps/workbench/Gemfile
+ 100644 blob b51f674d90f68bfb50d9304068f915e42b04aea4 2249 apps/workbench/Gemfile
+ 100755 blob cdd5ebaff27781f93ab85e484410c0ce9e97770f 1012 crunch_scripts/hash
+ EOS
+ Repository.any_instance.
+ stubs(:cat_file).with(fakesha1, fakefilename).returns fakefile
+ Repository.any_instance.
+ stubs(:show).with(fakesha1).returns fakecommit
+ return fakesha1, fakecommit, fakefile
+ end
+end
diff --git a/apps/workbench/test/integration/repositories_browse_test.rb b/apps/workbench/test/integration/repositories_browse_test.rb
new file mode 100644
index 0000000..147bf46
--- /dev/null
+++ b/apps/workbench/test/integration/repositories_browse_test.rb
@@ -0,0 +1,37 @@
+require 'integration_helper'
+require 'helpers/repository_stub_helper'
+require 'helpers/share_object_helper'
+
+class RepositoriesTest < ActionDispatch::IntegrationTest
+ include RepositoryStubHelper
+ include ShareObjectHelper
+
+ reset_api_fixtures :after_each_test, false
+
+ setup do
+ need_javascript
+ end
+
+ test "browse repository from jobs#show" do
+ sha1 = api_fixture('jobs')['running']['script_version']
+ _, fakecommit, fakefile =
+ stub_repo_content sha1: sha1, filename: 'crunch_scripts/hash'
+ show_object_using 'active', 'jobs', 'running', sha1
+ click_on api_fixture('jobs')['running']['script']
+ assert_text fakefile
+ click_on 'crunch_scripts'
+ assert_selector 'td a', text: 'hash'
+ click_on 'foo'
+ assert_selector 'td a', text: 'crunch_scripts'
+ click_on sha1
+ assert_text fakecommit
+
+ show_object_using 'active', 'jobs', 'running', sha1
+ click_on 'active/foo'
+ assert_selector 'td a', text: 'crunch_scripts'
+
+ show_object_using 'active', 'jobs', 'running', sha1
+ click_on sha1
+ assert_text fakecommit
+ end
+end
diff --git a/apps/workbench/test/test_helper.rb b/apps/workbench/test/test_helper.rb
index fdde55d..4ab162c 100644
--- a/apps/workbench/test/test_helper.rb
+++ b/apps/workbench/test/test_helper.rb
@@ -270,12 +270,17 @@ class ActiveSupport::TestCase
end
def after_teardown
- if self.class.want_reset_api_fixtures[:after_each_test]
+ if self.class.want_reset_api_fixtures[:after_each_test] and
+ @want_reset_api_fixtures != false
self.class.reset_api_fixtures_now
end
super
end
+ def reset_api_fixtures_after_test t=true
+ @want_reset_api_fixtures = t
+ end
+
protected
def self.reset_api_fixtures_now
# Never try to reset fixtures when we're just using test
diff --git a/services/api/test/fixtures/jobs.yml b/services/api/test/fixtures/jobs.yml
index ea6cbb0..cc1adfe 100644
--- a/services/api/test/fixtures/jobs.yml
+++ b/services/api/test/fixtures/jobs.yml
@@ -7,6 +7,8 @@ running:
created_at: <%= 3.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: ~
+ script: hash
+ repository: active/foo
script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
running: true
success: ~
@@ -31,6 +33,8 @@ running_cancelled:
created_at: <%= 4.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: ~
+ script: hash
+ repository: active/foo
script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
running: true
success: ~
@@ -56,6 +60,8 @@ uses_nonexistent_script_version:
created_at: <%= 5.minute.ago.to_s(:db) %>
started_at: <%= 3.minute.ago.to_s(:db) %>
finished_at: <%= 2.minute.ago.to_s(:db) %>
+ script: hash
+ repository: active/foo
running: false
success: true
output: d41d8cd98f00b204e9800998ecf8427e+0
commit b62aa0b5accf722383b0626686ba04e5a333274f
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Mar 31 10:05:47 2015 -0400
5416: Fix comment.
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index f3d1792..8fa31e7 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -202,5 +202,5 @@ common:
# in the directory where your API server is running.
anonymous_user_token: false
- # Include Accept-Encoding header when making API requests
+ # Enable response payload compression in Arvados API requests.
include_accept_encoding_header_in_api_requests: true
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list