[ARVADOS] created: 3ec81a139360378eedea7ac3044b2c05645c31c8
git at public.curoverse.com
git at public.curoverse.com
Fri Mar 21 17:10:28 EDT 2014
at 3ec81a139360378eedea7ac3044b2c05645c31c8 (commit)
commit 3ec81a139360378eedea7ac3044b2c05645c31c8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Mar 21 17:10:18 2014 -0400
Now fetches into a internal repository and runs crunch-dispatch from that repository.
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 9f1663d..e0e002f 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -433,6 +433,7 @@ class WhRunPipelineInstance
job = JobCache.create({:script => c[:script],
:script_parameters => c[:script_parameters],
:script_version => c[:script_version],
+ :repository => c[:repository],
:minimum_script_version => c[:minimum_script_version],
:exclude_script_versions => c[:exclude_minimum_script_versions],
:nondeterministic => c[:nondeterministic],
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 04a054b..012c7d0 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -154,7 +154,7 @@ class Dispatcher
arvados_internal = File.join(Rails.configuration.git_repositories_dir, '_arvados_internal')
if not File.exists? arvados_internal
- `mkdir -p #{arvados_internal.shellescape}' && cd #{arvados_internal.shellescape} && git init`
+ `mkdir -p #{arvados_internal.shellescape} && cd #{arvados_internal.shellescape} && git init`
end
src_repo = File.join(Rails.configuration.git_repositories_dir, job.repository, '.git')
commit d48911f3b11c4a8612256c74c1a3093c2d86240d
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Mar 21 15:29:32 2014 -0400
Added migration to add "repository" column.
Crunch-dispatch.rb will now fetch into a single repository.
Needs testing.
diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index 07daf71..b35bc4e 100644
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -49,7 +49,6 @@ class Arvados::V1::JobsController < ApplicationController
end
# Don't pass these on to activerecord
- resource_attrs.delete(:repository)
resource_attrs.delete(:minimum_script_version)
resource_attrs.delete(:exclude_script_versions)
resource_attrs.delete(:no_reuse)
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index 6e3f90c..4af3fa8 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -37,6 +37,7 @@ class Job < ArvadosModel
t.add :log_stream_href
t.add :log_buffer
t.add :nondeterministic
+ t.add :repository
end
def assert_finished
diff --git a/services/api/db/migrate/20140321191343_add_repository_column_to_job.rb b/services/api/db/migrate/20140321191343_add_repository_column_to_job.rb
new file mode 100644
index 0000000..e1ebb2e
--- /dev/null
+++ b/services/api/db/migrate/20140321191343_add_repository_column_to_job.rb
@@ -0,0 +1,9 @@
+class AddRepositoryColumnToJob < ActiveRecord::Migration
+ def up
+ add_column :jobs, :repository, :string
+ end
+
+ def down
+ remove_column :jobs, :repository
+ end
+end
diff --git a/services/api/db/schema.rb b/services/api/db/schema.rb
index 4d638b5..97e6e9c 100644
--- a/services/api/db/schema.rb
+++ b/services/api/db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20140319160547) do
+ActiveRecord::Schema.define(:version => 20140321191343) do
create_table "api_client_authorizations", :force => true do |t|
t.string "api_token", :null => false
@@ -190,6 +190,7 @@ ActiveRecord::Schema.define(:version => 20140319160547) do
t.text "tasks_summary"
t.text "runtime_constraints"
t.boolean "nondeterministic"
+ t.string "repository"
end
add_index "jobs", ["created_at"], :name => "index_jobs_on_created_at"
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 89306a0..04a054b 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -150,24 +150,23 @@ class Dispatcher
raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
end
+ require 'shellwords'
+
+ arvados_internal = File.join(Rails.configuration.git_repositories_dir, '_arvados_internal')
+ if not File.exists? arvados_internal
+ `mkdir -p #{arvados_internal.shellescape}' && cd #{arvados_internal.shellescape} && git init`
+ end
+
+ src_repo = File.join(Rails.configuration.git_repositories_dir, job.repository, '.git')
+ `cd #{arvados_internal.shellescape} && git fetch #{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
+
cmd_args << crunch_job_bin
cmd_args << '--job-api-token'
cmd_args << job_auth.api_token
cmd_args << '--job'
cmd_args << job.uuid
-
- cmd_args << '--git-dir'
- if File.exists?(File.
- join(Rails.configuration.git_repositories_dir,
- job.repository + '.git'))
- cmd_args << File.
- join(Rails.configuration.git_repositories_dir,
- job.repository + '.git')
- else
- cmd_args << File.
- join(Rails.configuration.git_repositories_dir,
- job.repository, '.git')
- end
+ cmd_args << '--git-dir'
+ cmd_args << arvados_internal
$stderr.puts "dispatch: #{cmd_args.join ' '}"
@@ -307,7 +306,7 @@ class Dispatcher
jobrecord = Job.find_by_uuid(job_done.uuid)
jobrecord.running = false
- jobrecord.finished_at ||= Time.now,
+ jobrecord.finished_at ||= Time.now
# Don't set 'jobrecord.success = false' because if the job failed to run due to an
# issue with crunch-job or slurm, we want the job to stay in the queue.
jobrecord.save!
commit 5d493f84601e6905f0cde5a03c6502db38198770
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Mar 21 14:55:02 2014 -0400
Removing need for commit table.
diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index c4a7bd6..89306a0 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -156,20 +156,17 @@ class Dispatcher
cmd_args << '--job'
cmd_args << job.uuid
- commit = Commit.where(sha1: job.script_version).first
- if commit
- cmd_args << '--git-dir'
- if File.exists?(File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name + '.git'))
- cmd_args << File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name + '.git')
- else
- cmd_args << File.
- join(Rails.configuration.git_repositories_dir,
- commit.repository_name, '.git')
- end
+ cmd_args << '--git-dir'
+ if File.exists?(File.
+ join(Rails.configuration.git_repositories_dir,
+ job.repository + '.git'))
+ cmd_args << File.
+ join(Rails.configuration.git_repositories_dir,
+ job.repository + '.git')
+ else
+ cmd_args << File.
+ join(Rails.configuration.git_repositories_dir,
+ job.repository, '.git')
end
$stderr.puts "dispatch: #{cmd_args.join ' '}"
commit 4d3a1c5346c99da360af58f1932b8f7a2ba87723
Merge: 5b03914 8e603cb
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Mar 20 15:38:51 2014 -0400
Merge remote-tracking branch 'origin/master' into 2051-nondeterministic-jobs
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list