[ARVADOS] updated: 12b3b4e1c343eec1d710bbded2383adaefc1b242
git at public.curoverse.com
git at public.curoverse.com
Tue Nov 18 11:31:23 EST 2014
Summary of changes:
apps/workbench/Gemfile | 2 +-
apps/workbench/Gemfile.lock | 12 +-
.../workbench/app/assets/javascripts/filterable.js | 41 +++++-
.../app/assets/javascripts/infinite_scroll.js | 7 +-
.../app/assets/javascripts/pipeline_instances.js | 10 +-
.../app/assets/javascripts/select_modal.js | 31 ++++-
.../app/assets/javascripts/selection.js.erb | 3 +-
.../app/controllers/application_controller.rb | 15 ++-
.../controllers/pipeline_instances_controller.rb | 26 ++--
apps/workbench/app/helpers/application_helper.rb | 11 +-
apps/workbench/app/models/arvados_resource_list.rb | 9 +-
.../app/views/application/_choose.html.erb | 9 +-
.../app/views/collections/_choose_rows.html.erb | 4 +-
.../collections/_show_chooser_preview.html.erb | 2 +-
.../app/views/collections/_show_files.html.erb | 23 +++-
.../app/views/layouts/application.html.erb | 2 +-
apps/workbench/app/views/layouts/body.html.erb | 13 --
.../views/pipeline_instances/_show_recent.html.erb | 44 +------
..._recent.html.erb => _show_recent_rows.html.erb} | 45 +------
.../app/views/pipeline_instances/index.html.erb | 14 ++-
.../pipeline_instances_controller_test.rb | 2 +-
.../integration/filterable_infinite_scroll_test.rb | 29 +++++
.../test/integration/pipeline_instances_test.rb | 119 +++++++++++++-----
.../test/unit/arvados_resource_list_test.rb | 11 ++
crunch_scripts/run-command | 4 +
doc/api/methods/jobs.html.textile.liquid | 6 +-
sdk/cli/bin/arv-run-pipeline-instance | 139 +++++++++++++--------
sdk/cli/bin/crunch-job | 91 ++++++++------
sdk/go/keepclient/keepclient_test.go | 10 +-
sdk/ruby/arvados.gemspec | 2 +-
.../api/app/controllers/application_controller.rb | 8 ++
.../app/controllers/arvados/v1/jobs_controller.rb | 84 +++++++------
services/api/app/models/node.rb | 60 ++++++---
services/api/config/application.default.yml | 22 +++-
services/api/config/unbound.template | 4 +
services/api/script/restart-dns-server | 34 +++++
services/api/script/restart-dnsmasq | 21 ----
services/api/test/fixtures/jobs.yml | 17 +++
services/api/test/fixtures/pipeline_templates.yml | 38 ++++++
.../arvados/v1/collections_controller_test.rb | 12 ++
.../arvados/v1/job_reuse_controller_test.rb | 81 ++++++++----
services/api/test/unit/job_test.rb | 11 +-
42 files changed, 744 insertions(+), 384 deletions(-)
copy apps/workbench/app/views/pipeline_instances/{_show_recent.html.erb => _show_recent_rows.html.erb} (58%)
create mode 100644 apps/workbench/test/integration/filterable_infinite_scroll_test.rb
create mode 100644 services/api/config/unbound.template
create mode 100755 services/api/script/restart-dns-server
delete mode 100755 services/api/script/restart-dnsmasq
via 12b3b4e1c343eec1d710bbded2383adaefc1b242 (commit)
via a1df01872c35932e945d48e2241aa86972c7f02c (commit)
via 1362e1c7a214ef788d4d9672b35f031937bd2d26 (commit)
via 14b762d2dd37a0a533532839aaf0488c11205e42 (commit)
via 1a520a700456325a8a7427d74fa11ed15b62a4b2 (commit)
via e82958936210b6bbc8cc04b237229fdf6fac295c (commit)
via 4805867bc1004ae9d755a6375d592748a7b29585 (commit)
via ff7b22c70cd77073d9bdbebac0bf03d43745ed0c (commit)
via 95a61743fd1df435313042c17e581d4b7b457a2a (commit)
via 78447c70f491db25dc99eca60ebcc1b7a5362ec3 (commit)
via 664919d58c3689cd9e0a25547ec1e02d9adda38c (commit)
via 97ddc11818d059e42989f302fa15db4c5e19d5b3 (commit)
via 9a46fb0033514db6dd5e8ab4ccf793f78458a734 (commit)
via bfd5100c4e0bed1910875099937169f1b0c8ce7b (commit)
via d9f45bd01d2f5691a9aaca4f1fc00465497ef00c (commit)
via 0adcabd351108e83b630e27c48ce85f2b3cb3e14 (commit)
via e0f4747a2b24af68b474bb2da12e8799440c9acc (commit)
via a128baa147df78cadb5c1657dfa740fad80570ea (commit)
via 43e019a1e37effa0ccde680b52a61fbba1f858d4 (commit)
via 7395af3b19b8fc422fe2456c6bead1e7f4c60989 (commit)
via a75ef0985d2a385165168cbb488cf97382308549 (commit)
via 11374252ee3c26240420bb3aa0d0433fad71731f (commit)
via 5d87bcae66870127445e5eb018a69d606e93e244 (commit)
via 83e73eda7123a6de741198d759841fd186fc2a11 (commit)
via 06afd90ff09a83982451733f1e39974299d9a2d9 (commit)
via 4b3222657e160eaf61459aac843e8377100a53e4 (commit)
via abe8fc4c5818165ffb696448c81b3bf1bb5dc1dd (commit)
via 4c110b5f4e8e92d613f70427f27bdfae2ea376c7 (commit)
via 2ac2528dc0e5b700a2640e28dc66e474db9ad329 (commit)
via 9cd99d3070cd69664a31f8e25fad68fe8b7c43cf (commit)
via 6de0a4ab0323e69a5499e59af03c9c5e6125fed7 (commit)
via 2070784e52d89bbcf636062532a8ce37629737db (commit)
via 59f750a109a706d29862530dc3d282db75736d38 (commit)
via 8b7ed36ff057c4a483f4ee8a1c66929425b366c6 (commit)
via f5a1a9210e7638babe33d16938246e93ff4dd334 (commit)
via 3a31350c6265cb1135d3d4d40af436aae91a9894 (commit)
via c805714d8bbc51adb32a3a3ae4b82ea41eaa0de0 (commit)
via 9325dabcc2b6c32a589ba86cc052946b4875eb1d (commit)
via 95323ba327cadff2d1cb8c3f38c5dbb65dbd3694 (commit)
via eb185aab22bd916981ea874fd82e6a1fecce7cd2 (commit)
via f81483a374fbdfe6d5f36e5f5e55f92455afa155 (commit)
via 1ac0e3da5fbd477bd2acbf5bd8baa3c86f08194b (commit)
via dff161c1eaaecd6b557e88709fdf5f0dcb52821b (commit)
via a0e70cf1da033f7d94e728ab919bd8cfcabf3743 (commit)
via c28b9b91028c90497455bfde188915982377b825 (commit)
via 76c9a858643680d90f9694d49c212a54fe5b759f (commit)
via bc4992f9e11cd0639cf6461736963d66830c5af8 (commit)
via 39e39addee0a0de050c2b4d1f88fb7d56da7436b (commit)
via 333fbec95590cfcb30cfff7dc61e6b8cc41edf18 (commit)
via b2610e066d569b323fe55ffee615e61ca4e461d1 (commit)
via 6c7bfabc03237419decad9674c90ea7af906b615 (commit)
via cac035d7a171b90f0dc714d17b500b4062c0c58b (commit)
via 5cfa1aab53ba24d800d2dfa9ac1d88477063ab7f (commit)
via 61f8ceeb1d9b0786507d44ddceffce43fb9d13ae (commit)
via 8e209b3cc2065c4a52d7bdcb2c4c651b3f450d0d (commit)
via e993707e7a20ec37f4023bcf709fc77adce3857d (commit)
via 2d10f584cafa6348a933a5714ec8a7744957005d (commit)
via 16527c125c469451d810951ded0860294950a1f0 (commit)
via cba838830cb4b47420573ba0187e0eebbc5b6f1f (commit)
via 7b4fc9f10ad0df6e532299dbfc57ad4ba97ee015 (commit)
via 9547602dd2a0daf680635cec3249da759d3fd274 (commit)
via 911c97b06e4578e7c74db814ae1587f23ef18604 (commit)
via 7c76d9218ef91b14ede8a06fe9a96a5016361e17 (commit)
via f628e38317c20513cb73f81633d6a7c0663b8fd5 (commit)
via 7d197c6f7a676ac0e73f855fb4e6a3cd103dc55c (commit)
via 008cec917d19fadc011fdb1d61d7b54ffe1d4dda (commit)
via b917e8a2ebf8c51729ecde1d3cc3396d2fcf0034 (commit)
via d3cb9460f0a30f9602508bed7507c906b8a6d91d (commit)
via 1491cdd65b8a3ef75f66516004bc0680651da201 (commit)
via 29e012f772b1c1d4c286e143b50a9de9b8a6918c (commit)
via a5b4a7e8da51d84fe200d88dc39bd5431bb60a11 (commit)
via 723afcb87896a7dcee99bb41b37ef368464f4f40 (commit)
via cffe9566efaa838ada850353dd6e51ec4cc3ebff (commit)
via 88d7932586705439e7a02b403e65202164920660 (commit)
via 1c0e350af2a30f4627ad879545c9c6fc6554d92a (commit)
via 81bfa2fd57ee5e4b0042d418115f0e50170ee5eb (commit)
via 6ac59725fedc0a958d60d85740b47deb25addb5b (commit)
via 974ad63fc24161cb228003f4ef4cc32d5da578ff (commit)
via 87567c9440a65a42fdde62e513b40dc5a8585fdf (commit)
from 67e6092c464fcce21eecf79b93366231820e7705 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 12b3b4e1c343eec1d710bbded2383adaefc1b242
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Nov 18 11:31:22 2014 -0500
4035: Wait for arv-mount to start. Improve error handling. Add comments.
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 3cdccae..a2ff50a 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -524,66 +524,100 @@ class WhRunPipelineInstance
if c[:tooldescription]
require 'shellwords'
- # command workflow language tool description
- mktmpdir do |mountdir|
+ require 'tmpdir'
+ # Support command workflow language tool description
+ Dir::mktmpdir do |mountdir|
begin
- `arv-mount #{mountdir}`
+ # Create a temporary mount in order to read the tool
+ # description from the collection. We do this because the tool
+ # description file can have external references to other files
+ # in the same collection.
+ `arv-mount --by-id #{mountdir}`
if $?.exitstatus == 0
+ n = 0
+ while not File.exist? "#{mountdir}/README"
+ sleep 0.1
+ n += 1
+ if n >= 10
+ raise IOError.new "arv-mount not ready"
+ end
+ end
+
require 'tempfile'
jobdesc_file = Tempfile.new('jobdesc')
begin
+ # Read the tool description file so we can check for Docker down below.
+ tooldesc = nil
File.open "#{mountdir}/#{c[:tooldescription]}" do |f|
tooldesc = JSON.load f
end
+
+ # Create a cwl job order from script_parameters and write
+ # to tempfile so cwltool can read it.
jobdesc_file.write({"inputs" => Hash[c[:script_parameters].map do |key, spec|
[key, spec[:value]]
end]}.to_json)
jobdesc_file.close
- args = JSON.load(`cwltool --conformance-test --no-container --basedir='$(task.keepmount)' #{mountdir}/#{c[:tooldescription].shellescape} #{jobdesc_file.path}`)
- script_parameters = {}
- script_parameters["command"] = args["args"]
- script_parameters["task.stdout"] = args["stdout"]
- script_parameters["task.stdin"] = args["stdin"]
-
- runtime_constraints = {}
- container = tooldesc["requirements"].andand["environment"].andand["container"]
- if container and container["type"] == "docker"
- runtime_constraints["docker_image"] = container["imageId"]
+ # Run common workflow language tool in "conformance test"
+ # mode, this outputs a JSON object with a fully instantiated
+ # command line based on the job order.
+ command = `cwltool --conformance-test --no-container --basedir='$(task.keepmount)' #{mountdir}/#{c[:tooldescription].shellescape} #{jobdesc_file.path}`
+ if $?.exitstatus == 0
+ args = JSON.load(command)
+ script_parameters = {}
+ script_parameters["command"] = args["args"]
+ script_parameters["task.stdout"] = args["stdout"] if args["stdout"]
+ script_parameters["task.stdin"] = args["stdin"] if args["stdin"]
+
+ # Pull out the Docker container specified in the tool
+ # document and set the Arvados runtime constraint.
+ runtime_constraints = {}
+ container = tooldesc["requirements"].andand["environment"].andand["container"]
+ if container and container["type"] == "docker"
+ runtime_constraints["docker_image"] = container["imageId"]
+ end
+
+ job = JobCache.create(@instance, cname, {
+ :script => "run-command",
+ :script_parameters => script_parameters,
+ :script_version => "a1df01872c35932e945d48e2241aa86972c7f02c",
+ :repository => "arvados",
+ :nondeterministic => c[:nondeterministic],
+ :runtime_constraints => runtime_constraints,
+ :owner_uuid => owner_uuid,
+ :is_locked_by_uuid => (@options[:run_jobs_here] ? owner_uuid : nil),
+ :submit_id => my_submit_id,
+ :state => (if @options[:run_jobs_here] then "Running" else "Queued" end)
+ }, {
+ # This is the right place to put these attributes when
+ # dealing with new API servers.
+ :minimum_script_version => c[:minimum_script_version],
+ :exclude_script_versions => c[:exclude_minimum_script_versions],
+ :find_or_create => (@instance[:properties][:run_options].andand[:enable_job_reuse] &&
+ !c[:nondeterministic]),
+ :filters => c[:filters]
+ })
+ else
+ err_msg = command
end
-
- job = JobCache.create(@instance, cname, {
- :script => "run-command",
- :script_parameters => script_parameters,
- :script_version => "master",
- :repository => "arvados",
- :nondeterministic => c[:nondeterministic],
- :runtime_constraints => runtime_constraints,
- :owner_uuid => owner_uuid,
- :is_locked_by_uuid => (@options[:run_jobs_here] ? owner_uuid : nil),
- :submit_id => my_submit_id,
- :state => (if @options[:run_jobs_here] then "Running" else "Queued" end)
- }, {
- # This is the right place to put these attributes when
- # dealing with new API servers.
- :minimum_script_version => c[:minimum_script_version],
- :exclude_script_versions => c[:exclude_minimum_script_versions],
- :find_or_create => (@instance[:properties][:run_options].andand[:enable_job_reuse] &&
- !c[:nondeterministic]),
- :filters => c[:filters]
- })
-
ensure
jobdesc_file.unlink
end
+ else
+ err_msg = $?
end
+ rescue IOError => e
+ err_msg = e
ensure
+ # harmless if the mount failed
`fusermount -u #{mountdir}`
end
- end
- else
- debuglog "component #{cname} new job failed with output #{tooldesc}", 0
- job_creation_failed += 1
+
+ if job.nil?
+ debuglog "component #{cname} new job failed #{err_msg}", 0
+ job_creation_failed += 1
+ end
end
else
job = JobCache.create(@instance, cname, {
commit a1df01872c35932e945d48e2241aa86972c7f02c
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Nov 18 09:59:30 2014 -0500
4035: Use arv-mount to read the tool description file instead of arv-get in order to support files with external references. Add $(task.keepmount) to run-command.
diff --git a/crunch_scripts/run-command b/crunch_scripts/run-command
index 13ae918..74e33ca 100755
--- a/crunch_scripts/run-command
+++ b/crunch_scripts/run-command
@@ -81,12 +81,16 @@ def sub_taskid(v):
def sub_jobsrc(v):
return os.environ['CRUNCH_SRC']
+def sub_task_keepmount(v):
+ return os.environ['TASK_KEEPMOUNT']
+
subst.default_subs["task.tmpdir"] = sub_tmpdir
subst.default_subs["task.outdir"] = sub_outdir
subst.default_subs["job.srcdir"] = sub_jobsrc
subst.default_subs["node.cores"] = sub_cores
subst.default_subs["job.uuid"] = sub_jobid
subst.default_subs["task.uuid"] = sub_taskid
+subst.default_subs["task.keepmount"] = sub_task_keepmount
class SigHandler(object):
def __init__(self):
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 51f82d6..3cdccae 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -525,58 +525,63 @@ class WhRunPipelineInstance
if c[:tooldescription]
require 'shellwords'
# command workflow language tool description
- tooldesc = JSON.load(`arv-get #{c[:tooldescription].shellescape}`)
- if $?.exitstatus == 0
- require 'tempfile'
- tooldesc_file = Tempfile.new('tooldesc')
- jobdesc_file = Tempfile.new('jobdesc')
+ mktmpdir do |mountdir|
begin
- tooldesc_file.write(tooldesc.to_json)
- tooldesc_file.close
-
- jobdesc_file.write({"inputs" => Hash[c[:script_parameters].map do |key, spec|
- [key, spec[:value]]
- end]}.to_json)
- jobdesc_file.close
-
- args = JSON.load(`cwltool --conformance-test --no-container --basedir=/keep #{tooldesc_file.path} #{jobdesc_file.path}`)
- script_parameters = {}
- script_parameters["command"] = args["args"]
- script_parameters["task.stdout"] = args["stdout"]
- script_parameters["task.stdin"] = args["stdin"]
-
- runtime_constraints = {}
- container = tooldesc["requirements"].andand["environment"].andand["container"]
- if container and container["type"] == "docker"
- runtime_constraints["docker_image"] = container["imageId"]
- end
-
- job = JobCache.create(@instance, cname, {
- :script => "run-command",
- :script_parameters => script_parameters,
- :script_version => "master",
- :repository => "arvados",
- :nondeterministic => c[:nondeterministic],
- :runtime_constraints => runtime_constraints,
- :owner_uuid => owner_uuid,
- :is_locked_by_uuid => (@options[:run_jobs_here] ? owner_uuid : nil),
- :submit_id => my_submit_id,
- :state => (if @options[:run_jobs_here] then "Running" else "Queued" end)
- }, {
- # This is the right place to put these attributes when
- # dealing with new API servers.
- :minimum_script_version => c[:minimum_script_version],
- :exclude_script_versions => c[:exclude_minimum_script_versions],
- :find_or_create => (@instance[:properties][:run_options].andand[:enable_job_reuse] &&
- !c[:nondeterministic]),
- :filters => c[:filters]
- })
+ `arv-mount #{mountdir}`
+ if $?.exitstatus == 0
+ require 'tempfile'
+ jobdesc_file = Tempfile.new('jobdesc')
+ begin
+ File.open "#{mountdir}/#{c[:tooldescription]}" do |f|
+ tooldesc = JSON.load f
+ end
+ jobdesc_file.write({"inputs" => Hash[c[:script_parameters].map do |key, spec|
+ [key, spec[:value]]
+ end]}.to_json)
+ jobdesc_file.close
+
+ args = JSON.load(`cwltool --conformance-test --no-container --basedir='$(task.keepmount)' #{mountdir}/#{c[:tooldescription].shellescape} #{jobdesc_file.path}`)
+ script_parameters = {}
+ script_parameters["command"] = args["args"]
+ script_parameters["task.stdout"] = args["stdout"]
+ script_parameters["task.stdin"] = args["stdin"]
+
+ runtime_constraints = {}
+ container = tooldesc["requirements"].andand["environment"].andand["container"]
+ if container and container["type"] == "docker"
+ runtime_constraints["docker_image"] = container["imageId"]
+ end
+ job = JobCache.create(@instance, cname, {
+ :script => "run-command",
+ :script_parameters => script_parameters,
+ :script_version => "master",
+ :repository => "arvados",
+ :nondeterministic => c[:nondeterministic],
+ :runtime_constraints => runtime_constraints,
+ :owner_uuid => owner_uuid,
+ :is_locked_by_uuid => (@options[:run_jobs_here] ? owner_uuid : nil),
+ :submit_id => my_submit_id,
+ :state => (if @options[:run_jobs_here] then "Running" else "Queued" end)
+ }, {
+ # This is the right place to put these attributes when
+ # dealing with new API servers.
+ :minimum_script_version => c[:minimum_script_version],
+ :exclude_script_versions => c[:exclude_minimum_script_versions],
+ :find_or_create => (@instance[:properties][:run_options].andand[:enable_job_reuse] &&
+ !c[:nondeterministic]),
+ :filters => c[:filters]
+ })
+
+ ensure
+ jobdesc_file.unlink
+ end
+ end
ensure
- tooldesc_file.unlink
- jobdesc_file.unlink
+ `fusermount -u #{mountdir}`
end
- else
+ end
+ else
debuglog "component #{cname} new job failed with output #{tooldesc}", 0
job_creation_failed += 1
end
commit 1362e1c7a214ef788d4d9672b35f031937bd2d26
Merge: 67e6092 14b762d
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Nov 18 09:43:48 2014 -0500
Merge branch 'master' into 4035-pipeline-support-cwl-tool
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list