[ARVADOS] created: dbbc879dec9ce61a64176925b8aeb1a38240817c

git at public.curoverse.com git at public.curoverse.com
Wed Apr 1 18:27:37 EDT 2015


        at  dbbc879dec9ce61a64176925b8aeb1a38240817c (commit)


commit dbbc879dec9ce61a64176925b8aeb1a38240817c
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Apr 1 18:28:19 2015 -0400

    5416: Browse git repository contents in workbench.

diff --git a/apps/workbench/app/controllers/repositories_controller.rb b/apps/workbench/app/controllers/repositories_controller.rb
index d32c92a..c5b3501 100644
--- a/apps/workbench/app/controllers/repositories_controller.rb
+++ b/apps/workbench/app/controllers/repositories_controller.rb
@@ -16,4 +16,20 @@ class RepositoriesController < ApplicationController
     panes.delete('Attributes') if !current_user.is_admin
     panes
   end
+
+  def show_tree
+    @commit = params[:commit]
+    @path = params[:path] || ''
+    @subtree = @object.ls_subtree @commit, @path.chomp('/')
+  end
+
+  def show_blob
+    @commit = params[:commit]
+    @path = params[:path]
+    @blobdata = @object.cat_file @commit, @path
+  end
+
+  def show_commit
+    @commit = params[:commit]
+  end
 end
diff --git a/apps/workbench/app/models/repository.rb b/apps/workbench/app/models/repository.rb
index b062dda..4d0430b 100644
--- a/apps/workbench/app/models/repository.rb
+++ b/apps/workbench/app/models/repository.rb
@@ -12,4 +12,105 @@ class Repository < ArvadosBase
       []
     end
   end
+
+  def show commit_sha1
+    refresh
+    run_git 'show', commit_sha1
+  end
+
+  def cat_file commit_sha1, path
+    refresh
+    run_git 'cat-file', 'blob', commit_sha1 + ':' + path
+  end
+
+  def ls_tree_lr commit_sha1
+    refresh
+    run_git 'ls-tree', '-l', '-r', commit_sha1
+  end
+
+  # subtree returns a list of files under the given path at the
+  # specified commit. Results are returned as an array of file nodes,
+  # where each file node is an array [file mode, blob sha1, file size
+  # in bytes, path relative to the given directory]. If the path is
+  # not found, [] is returned.
+  def ls_subtree commit, path
+    path = path.chomp '/'
+    subtree = []
+    ls_tree_lr(commit).each_line do |line|
+      mode, type, sha1, size, filepath = line.split
+      next if type != 'blob'
+      if filepath[0,path.length] == path and
+          (path == '' or filepath[path.length] == '/')
+        subtree << [mode.to_i(8), sha1, size.to_i,
+                    filepath[path.length,filepath.length]]
+      end
+    end
+    subtree
+  end
+
+  protected
+
+  # refresh fetches the latest repository content into the local
+  # cache. It is a no-op if it has already been run on this object:
+  # this (pretty much) avoids doing more than one remote git operation
+  # per Workbench request.
+  def refresh
+    run_git 'fetch', http_fetch_url, '+*:*' unless @fresh
+    @fresh = true
+  end
+
+  # http_fetch_url returns an http:// or https:// fetch-url which can
+  # accept arvados API token authentication. The API server currently
+  # advertises SSH fetch-urls, which work for users with SSH keys but
+  # are useless for fetching repository content into workbench itself.
+  def http_fetch_url
+    "https://git.#{uuid[0,5]}.arvadosapi.com/#{name}.git"
+  end
+
+  # run_git sets up the ARVADOS_API_TOKEN environment variable,
+  # creates a local git directory for this repository if necessary,
+  # executes "git --git-dir localgitdir {args to run_git}", and
+  # returns the output. It raises GitCommandError if git exits
+  # non-zero.
+  def run_git *cmd
+    if not @workdir
+      workdir = File.expand_path uuid+'.git', Rails.configuration.repository_cache
+      if not File.exists? workdir
+        FileUtils.mkdir_p Rails.configuration.repository_cache
+        [['git', 'init', '--bare', workdir],
+        ].each do |cmd|
+          system *cmd
+          raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0
+        end
+      end
+      @workdir = workdir
+    end
+    [['git', '--git-dir', @workdir, 'config', '--local',
+      "credential.#{http_fetch_url}.username", 'none'],
+     ['git', '--git-dir', @workdir, 'config', '--local',
+      "credential.#{http_fetch_url}.helper",
+      '!token(){ echo password="$ARVADOS_API_TOKEN"; }; token'],
+     ['git', '--git-dir', @workdir, 'config', '--local',
+           'http.sslVerify',
+           Rails.configuration.arvados_insecure_https ? 'false' : 'true'],
+     ].each do |cmd|
+      system *cmd
+      raise GitCommandError.new($?.to_s) unless $?.exitstatus == 0
+    end
+    env = {}.
+      merge(ENV).
+      merge('ARVADOS_API_TOKEN' => Thread.current[:arvados_api_token])
+    cmd = ['git', '--git-dir', @workdir] + cmd
+    io = IO.popen(env, cmd, err: [:child, :out])
+    output = io.read
+    io.close
+    # "If [io] is opened by IO.popen, close sets $?." --ruby 2.2.1 docs
+    unless $?.exitstatus == 0
+      raise GitCommandError.new(cmd.inspect + "\n" + output)
+    end
+    output
+  end
+
+  class GitCommandError < StandardError
+  end
 end
diff --git a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
index 018d49f..ab318e1 100644
--- a/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/_running_component.html.erb
@@ -96,6 +96,12 @@
         <div class="row">
           <div class="col-md-6">
             <table>
+              <% # link to repo tree/file only if the repo is readable
+                 # and the commit is a sha1
+                 repo =
+                 (/^[0-9a-f]{40}$/ =~ current_component[:script_version] and
+                 Repository.where(name: current_component[:repository]).first)
+                 %>
               <% [:script, :repository, :script_version, :supplied_script_version, :nondeterministic].each do |k| %>
                 <tr>
                   <td style="padding-right: 1em">
@@ -104,6 +110,12 @@
                   <td>
                     <% if current_component[k].nil? %>
                       (none)
+                    <% elsif k == :repository %>
+                      <%= link_to_if repo, current_component[k], show_repository_tree_path(id: repo.uuid, commit: current_component[:script_version], path: '/') %>
+                    <% elsif k == :script %>
+                      <%= link_to_if repo, current_component[k], show_repository_blob_path(id: repo.uuid, commit: current_component[:script_version], path: 'crunch_scripts/'+current_component[:script]) %>
+                    <% elsif k == :script_version %>
+                      <%= link_to_if repo, current_component[k], show_repository_commit_path(id: repo.uuid, commit: current_component[:script_version]) %>
                     <% else %>
                       <%= current_component[k] %>
                     <% end %>
diff --git a/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb b/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb
new file mode 100644
index 0000000..736c187
--- /dev/null
+++ b/apps/workbench/app/views/repositories/_repository_breadcrumbs.html.erb
@@ -0,0 +1,13 @@
+<div class="pull-right">
+  <span class="deemphasize">Browsing <%= @object.name %> repository at commit</span>
+  <%= link_to(@commit, show_repository_commit_path(id: @object.uuid, commit: @commit), title: 'show commit message') %>
+</div>
+<p>
+  <%= link_to(@object.name, show_repository_tree_path(id: @object.uuid, commit: @commit, path: '/'), title: 'show root directory of source tree') %>
+  <% parents = ''
+     (@path || '').split('/').each do |pathpart|
+     parents = parents + pathpart + '/'
+     %>
+    / <%= link_to pathpart, show_repository_tree_path(id: @object.uuid, commit: @commit, path: parents) %>
+  <% end %>
+</p>
diff --git a/apps/workbench/app/views/repositories/show_blob.html.erb b/apps/workbench/app/views/repositories/show_blob.html.erb
new file mode 100644
index 0000000..acc34d1
--- /dev/null
+++ b/apps/workbench/app/views/repositories/show_blob.html.erb
@@ -0,0 +1,13 @@
+<%= render partial: 'repository_breadcrumbs' %>
+
+<% if not @blobdata.valid_encoding? %>
+  <div class="alert alert-warning">
+    <p>
+      This file has an invalid text encoding, so it can't be shown
+      here.  (This probably just means it's a binary file, not a text
+      file.)
+    </p>
+  </div>
+<% else %>
+  <pre><%= @blobdata %></pre>
+<% end %>
diff --git a/apps/workbench/app/views/repositories/show_commit.html.erb b/apps/workbench/app/views/repositories/show_commit.html.erb
new file mode 100644
index 0000000..3690be6
--- /dev/null
+++ b/apps/workbench/app/views/repositories/show_commit.html.erb
@@ -0,0 +1,3 @@
+<%= render partial: 'repository_breadcrumbs' %>
+
+<pre><%= @object.show @commit %></pre>
diff --git a/apps/workbench/app/views/repositories/show_tree.html.erb b/apps/workbench/app/views/repositories/show_tree.html.erb
new file mode 100644
index 0000000..4e2fcec
--- /dev/null
+++ b/apps/workbench/app/views/repositories/show_tree.html.erb
@@ -0,0 +1,40 @@
+<%= render partial: 'repository_breadcrumbs' %>
+
+<table class="table table-condensed table-hover">
+  <thead>
+    <tr>
+      <th>File</th>
+      <th class="data-size">Size</th>
+    </tr>
+  </thead>
+  <tbody>
+    <% @subtree.each do |mode, sha1, size, subpath| %>
+      <tr>
+        <td>
+          <span style="opacity: 0.6">
+            <% pathparts = subpath.sub(/^\//, '').split('/')
+               basename = pathparts.pop
+               parents = @path
+               pathparts.each do |pathpart| %>
+              <% parents = parents + '/' + pathpart %>
+              <%= link_to pathpart, url_for(path: parents) %>
+              /
+            <% end %>
+          </span>
+          <%= link_to basename, url_for(action: :show_blob, path: parents + '/' + basename) %>
+        </td>
+        <td class="data-size">
+          <%= human_readable_bytes_html(size) %>
+        </td>
+      </tr>
+    <% end %>
+    <% if @subtree.empty? %>
+      <tr>
+        <td>
+          No files found.
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+  <tfoot></tfoot>
+</table>
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index 8fa31e7..5b2391f 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -138,8 +138,14 @@ common:
   default_openid_prefix: https://www.google.com/accounts/o8/id
   send_user_setup_notification_email: true
 
-  # Set user_profile_form_fields to enable and configure the user profile page.
-  # Default is set to false. A commented setting with full description is provided below.
+  # Scratch directory used by the remote repository browsing
+  # feature. If it doesn't exist, it (and any missing parents) will be
+  # created using mkdir_p.
+  repository_cache: <%= File.expand_path 'tmp/git', Rails.root %>
+
+  # Set user_profile_form_fields to enable and configure the user
+  # profile page. Default is set to false. A commented example with
+  # full description is provided below.
   user_profile_form_fields: false
 
   # Below is a sample setting of user_profile_form_fields config parameter.
diff --git a/apps/workbench/config/routes.rb b/apps/workbench/config/routes.rb
index 7ed02e7..44d7ded 100644
--- a/apps/workbench/config/routes.rb
+++ b/apps/workbench/config/routes.rb
@@ -26,6 +26,11 @@ ArvadosWorkbench::Application.routes.draw do
   resources :repositories do
     post 'share_with', on: :member
   end
+  # {format: false} prevents rails from treating "foo.png" as foo?format=png
+  get '/repositories/:id/tree/:commit' => 'repositories#show_tree'
+  get '/repositories/:id/tree/:commit/*path' => 'repositories#show_tree', as: :show_repository_tree, format: false
+  get '/repositories/:id/blob/:commit/*path' => 'repositories#show_blob', as: :show_repository_blob, format: false
+  get '/repositories/:id/commit/:commit' => 'repositories#show_commit', as: :show_repository_commit
   match '/logout' => 'sessions#destroy', via: [:get, :post]
   get '/logged_out' => 'sessions#index'
   resources :users do
diff --git a/apps/workbench/test/controllers/repositories_controller_test.rb b/apps/workbench/test/controllers/repositories_controller_test.rb
index f95bb77..25bf557 100644
--- a/apps/workbench/test/controllers/repositories_controller_test.rb
+++ b/apps/workbench/test/controllers/repositories_controller_test.rb
@@ -1,7 +1,9 @@
 require 'test_helper'
+require 'helpers/repository_stub_helper'
 require 'helpers/share_object_helper'
 
 class RepositoriesControllerTest < ActionController::TestCase
+  include RepositoryStubHelper
   include ShareObjectHelper
 
   [
@@ -62,4 +64,61 @@ class RepositoriesControllerTest < ActionController::TestCase
       end
     end
   end
+
+  ### Browse repository content
+
+  [:active, :spectator].each do |user|
+    test "show tree to #{user}" do
+      reset_api_fixtures_after_test false
+      sha1, _, _ = stub_repo_content
+      get :show_tree, {
+        id: api_fixture('repositories')['foo']['uuid'],
+        commit: sha1,
+      }, session_for(user)
+      assert_response :success
+      assert_select 'tr td a', 'COPYING'
+      assert_select 'tr td', '625 bytes'
+      assert_select 'tr td a', 'apps'
+      assert_select 'tr td a', 'workbench'
+      assert_select 'tr td a', 'Gemfile'
+      assert_select 'tr td', '33.7 KiB'
+    end
+
+    test "show commit to #{user}" do
+      reset_api_fixtures_after_test false
+      sha1, commit, _ = stub_repo_content
+      get :show_commit, {
+        id: api_fixture('repositories')['foo']['uuid'],
+        commit: sha1,
+      }, session_for(user)
+      assert_response :success
+      assert_select 'pre', h(commit)
+    end
+
+    test "show blob to #{user}" do
+      reset_api_fixtures_after_test false
+      sha1, _, filedata = stub_repo_content filename: 'COPYING'
+      get :show_blob, {
+        id: api_fixture('repositories')['foo']['uuid'],
+        commit: sha1,
+        path: 'COPYING',
+      }, session_for(user)
+      assert_response :success
+      assert_select 'pre', h(filedata)
+    end
+  end
+
+  ['', '/'].each do |path|
+    test "show tree with path '#{path}'" do
+      reset_api_fixtures_after_test false
+      sha1, _, _ = stub_repo_content filename: 'COPYING'
+      get :show_tree, {
+        id: api_fixture('repositories')['foo']['uuid'],
+        commit: sha1,
+        path: path,
+      }, session_for(:active)
+      assert_response :success
+      assert_select 'tr td', 'COPYING'
+    end
+  end
 end
diff --git a/apps/workbench/test/helpers/repository_stub_helper.rb b/apps/workbench/test/helpers/repository_stub_helper.rb
new file mode 100644
index 0000000..b7d0573
--- /dev/null
+++ b/apps/workbench/test/helpers/repository_stub_helper.rb
@@ -0,0 +1,33 @@
+module RepositoryStubHelper
+  # Supply some fake git content.
+  def stub_repo_content opts={}
+    fakesha1 = opts[:sha1] || 'abcdefabcdefabcdefabcdefabcdefabcdefabcd'
+    fakefilename = opts[:filename] || 'COPYING'
+    fakefilesrc = File.expand_path('../../../../../'+fakefilename, __FILE__)
+    fakefile = File.read fakefilesrc
+    fakecommit = <<-EOS
+      commit abcdefabcdefabcdefabcdefabcdefabcdefabcd
+      Author: Fake R <fake at example.com>
+      Date:   Wed Apr 1 11:59:59 2015 -0400
+
+          It's a fake commit.
+
+    EOS
+    Repository.any_instance.stubs(:ls_tree_lr).with(fakesha1).returns <<-EOS
+      100644 blob eec475862e6ec2a87554e0fca90697e87f441bf5     226    .gitignore
+      100644 blob acbd7523ed49f01217874965aa3180cccec89d61     625    COPYING
+      100644 blob d645695673349e3947e8e5ae42332d0ac3164cd7   11358    LICENSE-2.0.txt
+      100644 blob c7a36c355b4a2b94dfab45c9748330022a788c91     622    README
+      100644 blob dba13ed2ddf783ee8118c6a581dbf75305f816a3   34520    agpl-3.0.txt
+      100644 blob 9bef02bbfda670595750fd99a4461005ce5b8f12     695    apps/workbench/.gitignore
+      100644 blob b51f674d90f68bfb50d9304068f915e42b04aea4    2249    apps/workbench/Gemfile
+      100644 blob b51f674d90f68bfb50d9304068f915e42b04aea4    2249    apps/workbench/Gemfile
+      100755 blob cdd5ebaff27781f93ab85e484410c0ce9e97770f    1012    crunch_scripts/hash
+    EOS
+    Repository.any_instance.
+      stubs(:cat_file).with(fakesha1, fakefilename).returns fakefile
+    Repository.any_instance.
+      stubs(:show).with(fakesha1).returns fakecommit
+    return fakesha1, fakecommit, fakefile
+  end
+end
diff --git a/apps/workbench/test/integration/repositories_browse_test.rb b/apps/workbench/test/integration/repositories_browse_test.rb
new file mode 100644
index 0000000..147bf46
--- /dev/null
+++ b/apps/workbench/test/integration/repositories_browse_test.rb
@@ -0,0 +1,37 @@
+require 'integration_helper'
+require 'helpers/repository_stub_helper'
+require 'helpers/share_object_helper'
+
+class RepositoriesTest < ActionDispatch::IntegrationTest
+  include RepositoryStubHelper
+  include ShareObjectHelper
+
+  reset_api_fixtures :after_each_test, false
+
+  setup do
+    need_javascript
+  end
+
+  test "browse repository from jobs#show" do
+    sha1 = api_fixture('jobs')['running']['script_version']
+    _, fakecommit, fakefile =
+      stub_repo_content sha1: sha1, filename: 'crunch_scripts/hash'
+    show_object_using 'active', 'jobs', 'running', sha1
+    click_on api_fixture('jobs')['running']['script']
+    assert_text fakefile
+    click_on 'crunch_scripts'
+    assert_selector 'td a', text: 'hash'
+    click_on 'foo'
+    assert_selector 'td a', text: 'crunch_scripts'
+    click_on sha1
+    assert_text fakecommit
+
+    show_object_using 'active', 'jobs', 'running', sha1
+    click_on 'active/foo'
+    assert_selector 'td a', text: 'crunch_scripts'
+
+    show_object_using 'active', 'jobs', 'running', sha1
+    click_on sha1
+    assert_text fakecommit
+  end
+end
diff --git a/apps/workbench/test/test_helper.rb b/apps/workbench/test/test_helper.rb
index fdde55d..4ab162c 100644
--- a/apps/workbench/test/test_helper.rb
+++ b/apps/workbench/test/test_helper.rb
@@ -270,12 +270,17 @@ class ActiveSupport::TestCase
   end
 
   def after_teardown
-    if self.class.want_reset_api_fixtures[:after_each_test]
+    if self.class.want_reset_api_fixtures[:after_each_test] and
+        @want_reset_api_fixtures != false
       self.class.reset_api_fixtures_now
     end
     super
   end
 
+  def reset_api_fixtures_after_test t=true
+    @want_reset_api_fixtures = t
+  end
+
   protected
   def self.reset_api_fixtures_now
     # Never try to reset fixtures when we're just using test
diff --git a/services/api/test/fixtures/jobs.yml b/services/api/test/fixtures/jobs.yml
index ea6cbb0..cc1adfe 100644
--- a/services/api/test/fixtures/jobs.yml
+++ b/services/api/test/fixtures/jobs.yml
@@ -7,6 +7,8 @@ running:
   created_at: <%= 3.minute.ago.to_s(:db) %>
   started_at: <%= 3.minute.ago.to_s(:db) %>
   finished_at: ~
+  script: hash
+  repository: active/foo
   script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
   running: true
   success: ~
@@ -31,6 +33,8 @@ running_cancelled:
   created_at: <%= 4.minute.ago.to_s(:db) %>
   started_at: <%= 3.minute.ago.to_s(:db) %>
   finished_at: ~
+  script: hash
+  repository: active/foo
   script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
   running: true
   success: ~
@@ -56,6 +60,8 @@ uses_nonexistent_script_version:
   created_at: <%= 5.minute.ago.to_s(:db) %>
   started_at: <%= 3.minute.ago.to_s(:db) %>
   finished_at: <%= 2.minute.ago.to_s(:db) %>
+  script: hash
+  repository: active/foo
   running: false
   success: true
   output: d41d8cd98f00b204e9800998ecf8427e+0

commit b62aa0b5accf722383b0626686ba04e5a333274f
Author: Tom Clegg <tom at curoverse.com>
Date:   Tue Mar 31 10:05:47 2015 -0400

    5416: Fix comment.

diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
index f3d1792..8fa31e7 100644
--- a/apps/workbench/config/application.default.yml
+++ b/apps/workbench/config/application.default.yml
@@ -202,5 +202,5 @@ common:
   # in the directory where your API server is running.
   anonymous_user_token: false
 
-  # Include Accept-Encoding header when making API requests
+  # Enable response payload compression in Arvados API requests.
   include_accept_encoding_header_in_api_requests: true

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list