[ARVADOS] updated: ee86983a93c335413c508c5717a8707bbced93e6
git at public.curoverse.com
git at public.curoverse.com
Thu Mar 27 09:27:46 EDT 2014
Summary of changes:
doc/README.textile | 6 ++
sdk/cli/bin/arv-run-pipeline-instance | 1 +
.../app/controllers/arvados/v1/jobs_controller.rb | 1 -
services/api/app/models/commit.rb | 1 +
services/api/app/models/job.rb | 1 +
services/api/config/application.default.yml | 12 ++++-
services/api/config/application.yml.example | 7 +--
.../20140321191343_add_repository_column_to_job.rb | 9 +++
services/api/db/schema.rb | 3 +-
services/api/script/crunch-dispatch.rb | 55 ++++++++++++--------
10 files changed, 65 insertions(+), 31 deletions(-)
create mode 100644 services/api/db/migrate/20140321191343_add_repository_column_to_job.rb
via ee86983a93c335413c508c5717a8707bbced93e6 (commit)
via 2c81dc8f676cad13b61c47b89fffd3dea93055fb (commit)
via 392c382ddaf8ea4c0c4b6655c7f508de73274d12 (commit)
via e423bbb1f70a3244af625165357acea8afbbfc85 (commit)
via 33166913f14878fd317b9a07e038f8d294453b94 (commit)
via 24742cd39eff59df5d4ac17d7d39c437c79a456b (commit)
from 7a537556b3503bf957ad3da2bf27885518a33230 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit ee86983a93c335413c508c5717a8707bbced93e6
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Mar 27 09:27:13 2014 -0400
Merged branch 1741-internal-git-repo into 2333-crunch-dispatch-token-management
diff --git a/doc/README.textile b/doc/README.textile
index 23a82fe..f88d304 100644
--- a/doc/README.textile
+++ b/doc/README.textile
@@ -18,6 +18,12 @@ h2. 1. Generate HTML pages
arvados/doc$ rake
</pre>
+Alternately, to make the documentation browsable on the local filesystem:
+
+<pre>
+arvados/doc$ rake generate baseurl=$PWD/.site
+</pre>
+
h2. 2. Preview HTML pages
<pre>
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 9f1663d..e0e002f 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -433,6 +433,7 @@ class WhRunPipelineInstance
job = JobCache.create({:script => c[:script],
:script_parameters => c[:script_parameters],
:script_version => c[:script_version],
+ :repository => c[:repository],
:minimum_script_version => c[:minimum_script_version],
:exclude_script_versions => c[:exclude_minimum_script_versions],
:nondeterministic => c[:nondeterministic],
diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index 07daf71..b35bc4e 100644
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -49,7 +49,6 @@ class Arvados::V1::JobsController < ApplicationController
end
# Don't pass these on to activerecord
- resource_attrs.delete(:repository)
resource_attrs.delete(:minimum_script_version)
resource_attrs.delete(:exclude_script_versions)
resource_attrs.delete(:no_reuse)
diff --git a/services/api/app/models/commit.rb b/services/api/app/models/commit.rb
index 4954124..6239011 100644
--- a/services/api/app/models/commit.rb
+++ b/services/api/app/models/commit.rb
@@ -139,6 +139,7 @@ class Commit < ActiveRecord::Base
next if repo.match /^\./
git_dir = File.join(@gitdirbase,
repo.match(/\.git$/) ? repo : File.join(repo, '.git'))
+ next if git_dir == Rails.configuration.git_internal_dir
repo_name = repo.sub(/\.git$/, '')
@repositories[repo_name] = {git_dir: git_dir}
end
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index 6e3f90c..4af3fa8 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -37,6 +37,7 @@ class Job < ArvadosModel
t.add :log_stream_href
t.add :log_buffer
t.add :nondeterministic
+ t.add :repository
end
def assert_finished
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index e46ff5d..298b001 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -45,7 +45,17 @@ common:
secret_token: ~
uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
- git_repositories_dir: /var/cache/git
+ # Git repositories must be readable by api server, or you won't be
+ # able to submit crunch jobs. To pass the test suites, put a clone
+ # of the arvados tree in {git_repositories_dir}/arvados.git or
+ # {git_repositories_dir}/arvados/.git
+ git_repositories_dir: /var/lib/arvados/git
+
+ # This is a (bare) repository that stores commits used in jobs. When a job
+ # runs, the source commits are first fetched into this repository, then this
+ # repository is used to deploy to compute nodes. This should NOT be a
+ # subdirectory of {git_repositiories_dir}.
+ git_internal_dir: /var/lib/arvados/internal.git
# :none or :slurm_immediate
crunch_job_wrapper: :none
diff --git a/services/api/config/application.yml.example b/services/api/config/application.yml.example
index 488c4bd..9162fc4 100644
--- a/services/api/config/application.yml.example
+++ b/services/api/config/application.yml.example
@@ -37,10 +37,5 @@ test:
secret_token: <%= rand(2**512).to_s(36) %>
common:
-
- # Git repositories must be readable by api server, or you won't be
- # able to submit crunch jobs. To pass the test suites, put a clone
- # of the arvados tree in {git_repositories_dir}/arvados.git or
- # {git_repositories_dir}/arvados/.git
- #
#git_repositories_dir: /var/cache/git
+ #git_internal_dir: /var/cache/arvados/internal.git
diff --git a/services/api/db/migrate/20140321191343_add_repository_column_to_job.rb b/services/api/db/migrate/20140321191343_add_repository_column_to_job.rb
new file mode 100644
index 0000000..e1ebb2e
--- /dev/null
+++ b/services/api/db/migrate/20140321191343_add_repository_column_to_job.rb
@@ -0,0 +1,9 @@
+class AddRepositoryColumnToJob < ActiveRecord::Migration
+ def up
+ add_column :jobs, :repository, :string
+ end
+
+ def down
+ remove_column :jobs, :repository
+ end
+end
diff --git a/services/api/db/schema.rb b/services/api/db/schema.rb
index 4d638b5..97e6e9c 100644
--- a/services/api/db/schema.rb
+++ b/services/api/db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20140319160547) do
+ActiveRecord::Schema.define(:version => 20140321191343) do
create_table "api_client_authorizations", :force => true do |t|
t.string "api_token", :null => false
@@ -190,6 +190,7 @@ ActiveRecord::Schema.define(:version => 20140319160547) do
t.text "tasks_summary"
t.text "runtime_constraints"
t.boolean "nondeterministic"
+ t.string "repository"
end
add_index "jobs", ["created_at"], :name => "index_jobs_on_created_at"
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index c35c5de..d9db69f 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -150,27 +150,32 @@ class Dispatcher
raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
end
+ require 'shellwords'
+
+ arvados_internal = Rails.configuration.git_internal_dir
+ if not File.exists? arvados_internal
+ $stderr.puts `mkdir -p #{arvados_internal.shellescape} && cd #{arvados_internal.shellescape} && git init --bare`
+ end
+
+ src_repo = File.join(Rails.configuration.git_repositories_dir, job.repository + '.git')
+ src_repo = File.join(Rails.configuration.git_repositories_dir, job.repository, '.git') unless File.exists? src_repo
+
+ unless src_repo
+ $stderr.puts "dispatch: #{File.join Rails.configuration.git_repositories_dir, job.repository} doesn't exist"
+ sleep 1
+ untake(job)
+ next
+ end
+
+ $stderr.puts `cd #{arvados_internal.shellescape} && git fetch --no-tags #{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
+
cmd_args << crunch_job_bin
cmd_args << '--job-api-token'
cmd_args << job_auth.api_token
cmd_args << '--job'
cmd_args << job.uuid
-
- commit = Commit.where(sha1: job.script_version).first
- if commit
- cmd_args << '--git-dir'
- if File.exists?(File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name + '.git'))
- cmd_args << File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name + '.git')
- else
- cmd_args << File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name, '.git')
- end
- end
+ cmd_args << '--git-dir'
+ cmd_args << arvados_internal
$stderr.puts "dispatch: #{cmd_args.join ' '}"
commit 2c81dc8f676cad13b61c47b89fffd3dea93055fb
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Mar 27 08:54:53 2014 -0400
Fixed dangling 'end'
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 9bdc1c2..c35c5de 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -326,11 +326,9 @@ class Dispatcher
def update_pipelines
expire_tokens = @pipe_auth_tokens.dup
@todo_pipelines.each do |p|
- pipe_auth = (@pipe_auth_tokens[p.uuid] ||= ApiClientAuthorization.
- create(user: User.where('uuid=?', p.modified_by_user_uuid).first,
- api_client_id: 0))
- end
- pipe_auth = @pipe_auth_tokens[p.uuid]
+ pipe_auth = (@pipe_auth_tokens[p.uuid] ||= ApiClientAuthorization.
+ create(user: User.where('uuid=?', p.modified_by_user_uuid).first,
+ api_client_id: 0))
puts `export ARVADOS_API_TOKEN=#{pipe_auth.api_token} && arv-run-pipeline-instance --run-here --no-wait --instance #{p.uuid}`
expire_tokens.delete p.uuid
end
commit 392c382ddaf8ea4c0c4b6655c7f508de73274d12
Merge: e423bbb 7a53755
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Mar 27 08:41:10 2014 -0400
Merge branch 'master' into 2333-crunch-dispatch-token-management
commit e423bbb1f70a3244af625165357acea8afbbfc85
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Mar 27 08:41:01 2014 -0400
Stylistic changes based on comments from Tom/Brett
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index ac1c428..9bdc1c2 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -39,7 +39,6 @@ class Dispatcher
def refresh_todo
@todo = Job.queue
@todo_pipelines = PipelineInstance.queue
- @pipe_auth_tokens ||= { }
end
def sinfo
@@ -327,12 +326,9 @@ class Dispatcher
def update_pipelines
expire_tokens = @pipe_auth_tokens.dup
@todo_pipelines.each do |p|
- if @pipe_auth_tokens[p.uuid].nil?
- pipe_auth = ApiClientAuthorization.
- new(user: User.where('uuid=?', p.modified_by_user_uuid).first,
- api_client_id: 0)
- pipe_auth.save
- @pipe_auth_tokens[p.uuid] = pipe_auth
+ pipe_auth = (@pipe_auth_tokens[p.uuid] ||= ApiClientAuthorization.
+ create(user: User.where('uuid=?', p.modified_by_user_uuid).first,
+ api_client_id: 0))
end
pipe_auth = @pipe_auth_tokens[p.uuid]
puts `export ARVADOS_API_TOKEN=#{pipe_auth.api_token} && arv-run-pipeline-instance --run-here --no-wait --instance #{p.uuid}`
@@ -348,6 +344,7 @@ class Dispatcher
def run
act_as_system_user
@running ||= {}
+ @pipe_auth_tokens ||= { }
$stderr.puts "dispatch: ready"
while !$signal[:term] or @running.size > 0
read_pipes
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list