[ARVADOS] updated: 0ce7d041f53bea47f98f509ccf7c196b8ad393b0
git at public.curoverse.com
git at public.curoverse.com
Tue Mar 25 15:25:04 EDT 2014
Summary of changes:
apps/workbench/Gemfile | 9 +-
apps/workbench/Gemfile.lock | 31 +++
apps/workbench/README.rdoc | 261 --------------------
apps/workbench/README.textile | 17 ++
apps/workbench/app/assets/javascripts/editable.js | 9 +
.../api_client_authorizations_controller.rb | 9 +-
.../app/controllers/application_controller.rb | 14 +-
.../app/controllers/collections_controller.rb | 14 +-
.../controllers/pipeline_templates_controller.rb | 5 +-
apps/workbench/app/models/arvados_api_client.rb | 26 ++-
apps/workbench/app/models/arvados_base.rb | 27 ++-
apps/workbench/app/models/arvados_resource_list.rb | 19 ++
.../app/views/application/_paging.html.erb | 126 ++++++++++
.../app/views/application/_show_recent.html.erb | 4 +
.../app/views/collections/_show_recent.html.erb | 6 +-
.../app/views/groups/_show_recent.html.erb | 4 +
.../app/views/layouts/application.html.erb | 2 +-
.../views/pipeline_instances/_show_recent.html.erb | 4 +
.../views/pipeline_templates/_show_recent.html.erb | 4 +
apps/workbench/config/application.default.yml | 12 +-
.../config/environments/development.rb.example | 20 --
.../config/environments/production.rb.example | 19 --
apps/workbench/config/environments/test.rb.example | 19 --
.../integration/api_client_authorizations_test.rb | 11 +
apps/workbench/test/integration/logins_test.rb | 14 +
.../test/integration/virtual_machines_test.rb | 17 ++
apps/workbench/test/integration_helper.rb | 68 +++++
.../install-workbench-app.html.textile.liquid | 2 -
.../job-and-pipeline-reference.html.textile.liquid | 2 +-
.../tutorials/tutorial-keep.html.textile.liquid | 2 +-
sdk/cli/bin/arv-run-pipeline-instance | 3 +-
sdk/cli/bin/crunch-job | 65 ++++-
.../api/app/controllers/application_controller.rb | 26 ++-
.../controllers/arvados/v1/schema_controller.rb | 11 +-
services/api/app/models/commit.rb | 67 ++----
...0547_separate_repository_from_script_version.rb | 1 +
services/api/script/crunch-dispatch.rb | 12 +-
services/api/script/rails | 4 -
.../arvados/v1/job_reuse_controller_test.rb | 8 +
.../functional/arvados/v1/jobs_controller_test.rb | 4 +
services/api/test/integration/permissions_test.rb | 244 ++++++++++++++++++
41 files changed, 782 insertions(+), 440 deletions(-)
delete mode 100644 apps/workbench/README.rdoc
create mode 100644 apps/workbench/README.textile
create mode 100644 apps/workbench/app/views/application/_paging.html.erb
create mode 100644 apps/workbench/test/integration/api_client_authorizations_test.rb
create mode 100644 apps/workbench/test/integration/logins_test.rb
create mode 100644 apps/workbench/test/integration/virtual_machines_test.rb
create mode 100644 apps/workbench/test/integration_helper.rb
create mode 100644 services/api/test/integration/permissions_test.rb
via 0ce7d041f53bea47f98f509ccf7c196b8ad393b0 (commit)
via eff1d4ece26ce459c1f19b202a2048b2f663bddc (commit)
via 4995783a3270e2f6d2d3b5226238fbbccf2864c1 (commit)
via a8537e43ce618b6f71e7195a3c8d08c5c8909ad1 (commit)
via 6f1ecb3df81614ff2007088e55ca4ce51e6ae90b (commit)
via 90f5ffab02a256828d43e6c04b511b47fa6ea2a9 (commit)
via 9e3d3a05f976ff795418ebdcc480f1a7afd9d4de (commit)
via b8b693c0c8db03a251d51eb42b46ea1faf719b1a (commit)
via e5a9f73847bab2d27a9423bb1c237063f7739bef (commit)
via c19e37b1452cc7c8234f05f2b0f2ce3ae2f0dff6 (commit)
via 2e2ce40b47b9e248805dcccc59baa77591c13d79 (commit)
via c6ff7517791480b4bf45de0a64553c97c5d4fd17 (commit)
via 8fa6952e7aaf9afc6d72699b165a693962151a21 (commit)
via 027953372c41b16b191d9a1124b82f1c21a759f9 (commit)
via f14fb304f537a3e006e4312f225c88217634eb6d (commit)
via 81c6881710e5d2e21f44682455f176dd965684cd (commit)
via 14ea55f217bbd7061fafc1fe69f4f453ad7c3e59 (commit)
via 354367a4c824a1e44192f2ea9364fd9414162f4d (commit)
via c36fe93c282ef320e7e5f0f9182994b50f7e88d3 (commit)
via 259c3b7b2e77b17590beef58b519f57a7d9b41e7 (commit)
via 40f9e6c22b1a4f7cbe473bae2e4336ef23feb41a (commit)
via 694e7cdb8bce9e18ca63fab78c3201232b8aab77 (commit)
via 3e3ef3aec33b4374a0d5cdfdb52b233499f9eb90 (commit)
via d4f3fdcb1cb4eea905961228eb46e9c8203c719c (commit)
via 46e1645bb91debb3e4ad98b82895ada5329e9351 (commit)
via df8de1bd9a517a1a3bfac0f8ab9ace41524d4b39 (commit)
via 7c8f8fda0c43bc379995ebb0204112eccfb6d97d (commit)
via 49016b4c5681cf819b3e42370e6e65c254cde4b4 (commit)
via 31506905a346552d6ef7a0b100c5fd203da7c2d8 (commit)
via 061dc0bcb7ced7ad196c6cc71dd25aab59948e46 (commit)
via 4d3a1c5346c99da360af58f1932b8f7a2ba87723 (commit)
via 8e603cbbfd5541cecf78967dc809c1676a607a09 (commit)
via aa19561b8139abbd27cea7bb88ec600b3df16de1 (commit)
via bdb71f7762196e179e495bda28f4375e5c69af31 (commit)
via 95e59f8e8c427bf199445a75102f1dfaeb4bf8a6 (commit)
via c50aa22b70789edf34fd72741f02f48ea138d5fe (commit)
via 764a15cfda6fc6c612a64f9b8542c408afbda921 (commit)
via e348aa5023875dbbbd40a0dc2ee32ea6aee53edd (commit)
via 5a5b2ae2ce1f5898bacb913459ecbc2d575733af (commit)
via 36b981fd2ddc6c491a9da1c5410ce802ecb13e65 (commit)
via 366c5db829a38cfceb5894a4af5ee4c3ec5901c8 (commit)
via 32f8186abb06258d0cfe0f05e040365c4ec6b68a (commit)
via 29ad4d318197a0e58ba0aa070723f19ee219364d (commit)
via e6c5563c60df03d4d061a6b301b730f0672b69a5 (commit)
via e63c37081683a418bf1ad7b9428ffaf24fe6a8f1 (commit)
via 44e147b16322f9bdd4606cda7deec631b951ab06 (commit)
via 18358de3cd35ef29e8826f45897a3373a79de291 (commit)
via 338bd17ff7567dca951a96192564537eca1c974d (commit)
from 5b039149732231ec9bd53fd8b5b005babf9eb82e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 0ce7d041f53bea47f98f509ccf7c196b8ad393b0
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Mar 25 15:24:55 2014 -0400
Removed trailing whitespace.
diff --git a/services/api/app/models/commit.rb b/services/api/app/models/commit.rb
index c371ac2..0e9ae11 100644
--- a/services/api/app/models/commit.rb
+++ b/services/api/app/models/commit.rb
@@ -4,7 +4,7 @@ class Commit < ActiveRecord::Base
def self.find_commit_range(current_user, repository, minimum, maximum, exclude)
# disallow starting with '-' so verision strings can't be interpreted as command line options
valid_pattern = /[A-Za-z0-9_][A-Za-z0-9_-]/
- if (minimum and !minimum.match valid_pattern) || !maximum.match valid_pattern
+ if (minimum and !minimum.match valid_pattern) or !maximum.match valid_pattern
logger.warn "find_commit_range called with string containing invalid characters: '#{minimum}', '#{maximum}'"
return nil
end
@@ -12,7 +12,7 @@ class Commit < ActiveRecord::Base
if minimum and minimum.empty?
minimum = nil
end
-
+
if !maximum
maximum = "HEAD"
end
@@ -65,7 +65,7 @@ class Commit < ActiveRecord::Base
end
end
- if minimum
+ if minimum
# Get the commit hash for the lower bound
min_hash = nil
IO.foreach("|git rev-list --max-count=1 #{minimum}") do |line|
@@ -74,12 +74,12 @@ class Commit < ActiveRecord::Base
# If not found or string is invalid, nothing else to do
next if !min_hash or !min_hash.match valid_pattern
-
+
# Now find all commits between them
#puts "git rev-list #{min_hash}..#{max_hash}"
IO.foreach("|git rev-list #{min_hash}..#{max_hash}") do |line|
hash = line.strip
- commits.push(hash) if !resolved_exclude or !resolved_exclude.include? hash
+ commits.push(hash) if !resolved_exclude or !resolved_exclude.include? hash
end
commits.push(min_hash) if !resolved_exclude or !resolved_exclude.include? min_hash
commit eff1d4ece26ce459c1f19b202a2048b2f663bddc
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Mar 25 15:24:15 2014 -0400
Fixed tests
Improved version string validation in find_commit_range
Removed spurious debugging flags
Removed dead code from commit.rb
Removed trailing whitespace
Fixed capitalization of Arvados in documentation
diff --git a/doc/user/reference/job-and-pipeline-reference.html.textile.liquid b/doc/user/reference/job-and-pipeline-reference.html.textile.liquid
index e9c24ec..56f4aec 100644
--- a/doc/user/reference/job-and-pipeline-reference.html.textile.liquid
+++ b/doc/user/reference/job-and-pipeline-reference.html.textile.liquid
@@ -10,7 +10,7 @@ table(table table-bordered table-condensed).
|_. Attribute |_. Type|_. Accepted values |_. Required|_. Description|
|script |string |filename |yes |The actual script that will be run by crunch. Must be the name of an executable file in the crunch_scripts/ directory at the git revision specified by script_version.|
|script_version |string |git branch, tag, or version hash |yes |The code version to run, which is available in the specified repository. May be a git hash or tag to specify an exact version, or a branch. If it is a branch, use the branch head.|
-|repository |string |name of git repository hosted by arvados |yes |The repository to search for script_version.|
+|repository |string |name of git repository hosted by Arvados |yes |The repository to search for script_version.|
|script_parameters |object |any JSON object |yes |The input parameters for the job, with the parameter names as keys mapping to parameter values.|
|minimum_script_version |string |git branch, tag, or version hash |no |The minimum acceptable script version when deciding whether to re-use a past job.|
|exclude_script_versions|array of strings|git branch, tag, or version hash|no |Script versions to exclude when deciding whether to re-use a past job.|
diff --git a/services/api/app/models/commit.rb b/services/api/app/models/commit.rb
index 7979ff4..c371ac2 100644
--- a/services/api/app/models/commit.rb
+++ b/services/api/app/models/commit.rb
@@ -1,47 +1,10 @@
class Commit < ActiveRecord::Base
require 'shellwords'
- # Make sure the specified commit really exists, and return the full
- # sha1 commit hash.
- #
- # Accepts anything "git rev-list" accepts, optionally (and
- # preferably) preceded by "repo_name:".
- #
- # Examples: "1234567", "master", "apps:1234567", "apps:master",
- # "apps:HEAD"
-
- # def self.find_by_commit_ish(commit_ish)
- # if only_valid_chars.match(commit_ish)
- # logger.warn "find_by_commit_ish called with string containing invalid characters: '#{commit_ish}'"
- # return nil
- # end
-
- # want_repo = nil
- # if commit_ish.index(':')
- # want_repo, commit_ish = commit_ish.split(':',2)
- # end
- # repositories.each do |repo_name, repo|
- # next if want_repo and want_repo != repo_name
- # ENV['GIT_DIR'] = repo[:git_dir]
- # # we're passing user input to a command line, this is a potential a security hole but I am reasonably confident that shellescape sanitizes the input adequately
- # IO.foreach("|git rev-list --max-count=1 --format=oneline 'origin/'#{commit_ish.shellescape} 2>/dev/null || git rev-list --max-count=1 --format=oneline ''#{commit_ish.shellescape}") do |line|
- # sha1, message = line.strip.split " ", 2
- # next if sha1.length != 40
- # begin
- # Commit.find_or_create_by_repository_name_and_sha1_and_message(repo_name, sha1, message[0..254])
- # rescue
- # logger.warn "find_or_create failed: repo_name #{repo_name} sha1 #{sha1} message #{message[0..254]}"
- # # Ignore cache failure. Commit is real. We should proceed.
- # end
- # return sha1
- # end
- # end
- # nil
- # end
-
def self.find_commit_range(current_user, repository, minimum, maximum, exclude)
- only_valid_chars = /[^A-Za-z0-9_-]/
- if only_valid_chars.match(minimum) || only_valid_chars.match(maximum)
+ # disallow starting with '-' so verision strings can't be interpreted as command line options
+ valid_pattern = /[A-Za-z0-9_][A-Za-z0-9_-]/
+ if (minimum and !minimum.match valid_pattern) || !maximum.match valid_pattern
logger.warn "find_commit_range called with string containing invalid characters: '#{minimum}', '#{maximum}'"
return nil
end
@@ -87,16 +50,18 @@ class Commit < ActiveRecord::Base
max_hash = line.strip
end
- # If not found, nothing else to do
- next if !max_hash
+ # If not found or string is invalid, nothing else to do
+ next if !max_hash or !max_hash.match valid_pattern
resolved_exclude = nil
if exclude
resolved_exclude = []
exclude.each do |e|
- IO.foreach("|git rev-list --max-count=1 #{e}") do |line|
- resolved_exclude.push(line.strip)
- end
+ if e.match valid_pattern
+ IO.foreach("|git rev-list --max-count=1 #{e}") do |line|
+ resolved_exclude.push(line.strip)
+ end
+ end
end
end
@@ -107,8 +72,8 @@ class Commit < ActiveRecord::Base
min_hash = line.strip
end
- # If not found, nothing else to do
- next if !min_hash
+ # If not found or string is invalid, nothing else to do
+ next if !min_hash or !min_hash.match valid_pattern
# Now find all commits between them
#puts "git rev-list #{min_hash}..#{max_hash}"
diff --git a/services/api/db/migrate/20140319160547_separate_repository_from_script_version.rb b/services/api/db/migrate/20140319160547_separate_repository_from_script_version.rb
index 6dac829..e740987 100644
--- a/services/api/db/migrate/20140319160547_separate_repository_from_script_version.rb
+++ b/services/api/db/migrate/20140319160547_separate_repository_from_script_version.rb
@@ -26,5 +26,6 @@ class SeparateRepositoryFromScriptVersion < ActiveRecord::Migration
end
def down
+ raise ActiveRecord::IrreversibleMigration
end
end
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index b6e0f67..6df9a19 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -330,7 +330,7 @@ class Dispatcher
api_client_id: 0)
pipe_auth.save
- puts `export ARVADOS_API_TOKEN=#{pipe_auth.api_token} && arv-run-pipeline-instance --run-here --no-wait --debug --debug-level=3 --instance #{p.uuid}`
+ puts `export ARVADOS_API_TOKEN=#{pipe_auth.api_token} && arv-run-pipeline-instance --run-here --no-wait --instance #{p.uuid}`
end
end
diff --git a/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb b/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
index 220566b..03e577f 100644
--- a/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
@@ -13,6 +13,7 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
post :create, job: {
script: "hash",
script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
script_parameters: {
input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
an_integer: '1'
@@ -32,6 +33,7 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
script: "hash",
minimum_script_version: "tag1",
script_version: "master",
+ repository: "foo",
script_parameters: {
input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
an_integer: '1'
@@ -50,6 +52,7 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
post :create, job: {
script: "hash",
script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
script_parameters: {
input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
an_integer: '2'
@@ -68,6 +71,7 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
post :create, job: {
script: "hash",
script_version: "master",
+ repository: "foo",
script_parameters: {
input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
an_integer: '2'
@@ -86,6 +90,7 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
post :create, job: {
script: "hash",
script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
script_parameters: {
input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
an_integer: '1'
@@ -105,6 +110,7 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
post :create, job: {
script: "hash2",
script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
script_parameters: {
input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
an_integer: '1'
@@ -123,6 +129,7 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
post :create, job: {
script: "hash",
script_version: "4fe459abe02d9b365932b8f5dc419439ab4e2577",
+ repository: "foo",
script_parameters: {
input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
an_integer: '1'
@@ -142,6 +149,7 @@ class Arvados::V1::JobReuseControllerTest < ActionController::TestCase
script: "hash",
minimum_script_version: "31ce37fe365b3dc204300a3e4c396ad333ed0556",
script_version: "master",
+ repository: "foo",
exclude_script_versions: ["tag1"],
script_parameters: {
input: 'fa7aeb5140e2848d39b416daeef4ffc5+45',
diff --git a/services/api/test/functional/arvados/v1/jobs_controller_test.rb b/services/api/test/functional/arvados/v1/jobs_controller_test.rb
index 3b6b786..a90b83f 100644
--- a/services/api/test/functional/arvados/v1/jobs_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/jobs_controller_test.rb
@@ -3,11 +3,14 @@ load 'test/functional/arvados/v1/git_setup.rb'
class Arvados::V1::JobsControllerTest < ActionController::TestCase
+ include GitSetup
+
test "submit a job" do
authorize_with :active
post :create, job: {
script: "hash",
script_version: "master",
+ repository: "foo",
script_parameters: {}
}
assert_response :success
@@ -23,6 +26,7 @@ class Arvados::V1::JobsControllerTest < ActionController::TestCase
script: "hash",
script_version: "master",
script_parameters: {},
+ repository: "foo",
started_at: Time.now,
finished_at: Time.now,
running: false,
commit 4995783a3270e2f6d2d3b5226238fbbccf2864c1
Merge: 4d3a1c5 a8537e4
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Mar 25 14:40:01 2014 -0400
Merge branch 'master' into 2051-nondeterministic-jobs
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list