[ARVADOS] updated: b402e6a66e569167c0e990a4fdfc2843df76afd8
git at public.curoverse.com
git at public.curoverse.com
Fri May 30 16:22:08 EDT 2014
Summary of changes:
...rial-gatk-variantfiltration.html.textile.liquid | 42 +----------
doc/user/topics/tutorial-job1.html.textile.liquid | 12 +---
sdk/python/arvados/collection.py | 21 +++++-
sdk/python/arvados/commands/put.py | 4 +-
sdk/python/test_cmdline.py | 81 ++++++++++++++++++++++
sdk/python/tests/test_keep_client.py | 20 ++++--
.../app/controllers/arvados/v1/jobs_controller.rb | 45 ------------
services/api/app/models/job.rb | 19 -----
services/api/config/application.yml.example | 10 +--
services/api/config/routes.rb | 1 -
.../arvados/v1/collections_controller_test.rb | 14 ++++
11 files changed, 140 insertions(+), 129 deletions(-)
create mode 100644 sdk/python/test_cmdline.py
via b402e6a66e569167c0e990a4fdfc2843df76afd8 (commit)
via 070e2e8e80cfc88414dd2595cae5bfddce1cd94f (commit)
via b6ea1fe3bf38bf28823c80b3aef98239a1c0311b (commit)
via c1e7255a85dfc2807ba78e1cf9d109d896c80b42 (commit)
via 39b2ed3024860262747fd6622433e0bf7ba39530 (commit)
via 5b15dc0308bd3ddcc3c50828269888b5811d0007 (commit)
via 3107d80e87c4fd8a4c6f6e06d9a22537bc1f19b0 (commit)
via b2812e9406253ded204ac853fb47437e8118bcdd (commit)
via 9d4f6f0284011fbc7a8edd203d1fab93a9a61321 (commit)
via 30b6c5a0f8dc91c76ca30ba4b5263b7eab858bdb (commit)
via 46460c9615f07184ecb9917e4c948635712859da (commit)
from ff3686b372cd598ee990acd990af2b04daf91ada (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b402e6a66e569167c0e990a4fdfc2843df76afd8
Merge: 070e2e8 b6ea1fe
Author: radhika <radhika at curoverse.com>
Date: Fri May 30 16:21:49 2014 -0400
Merge branch 'master' into 2895-no-more-redis
commit 070e2e8e80cfc88414dd2595cae5bfddce1cd94f
Author: radhika <radhika at curoverse.com>
Date: Fri May 30 16:20:19 2014 -0400
2895: remove jobs.log_tail_follow api since we are now using websockets instead of redis
diff --git a/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid b/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
index 0248325..ea608b4 100644
--- a/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
+++ b/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
@@ -154,48 +154,8 @@ Now start a job:
"5ee633fe2569d2a42dd81b07490d5d13+82",
"c905c8d8443a9c44274d98b7c6cfaa32+94",
"d237a90bae3870b3b033aea1e99de4a9+10820"
- ],
- "log_stream_href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-n9k7qyp7bs5b9d4/log_tail_follow"
+ ]
}
-~$ <span class="userinput">arv job log_tail_follow --uuid qr1hi-8i9sb-n9k7qyp7bs5b9d4</span>
-Tue Dec 17 19:02:16 2013 salloc: Granted job allocation 1251
-Tue Dec 17 19:02:17 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 check slurm allocation
-Tue Dec 17 19:02:17 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 node compute13 - 8 slots
-Tue Dec 17 19:02:17 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 start
-Tue Dec 17 19:02:17 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 Install revision 76588bfc57f33ea1b36b82ca7187f465b73b4ca4
-Tue Dec 17 19:02:18 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 Clean-work-dir exited 0
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 Install exited 0
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 script GATK2-VariantFiltration
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 script_version 76588bfc57f33ea1b36b82ca7187f465b73b4ca4
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 script_parameters {"input":"5ee633fe2569d2a42dd81b07490d5d13+82","gatk_bundle":"d237a90bae3870b3b033aea1e99de4a9+10820","gatk_binary_tarball":"c905c8d8443a9c44274d98b7c6cfaa32+94"}
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 runtime_constraints {"max_tasks_per_node":0}
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 start level 0
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 status: 0 done, 0 running, 1 todo
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 0 job_task qr1hi-ot0gb-d3sjxerucfbvyev
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 0 child 4946 started on compute13.1
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 status: 0 done, 1 running, 0 todo
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 0 child 4946 on compute13.1 exit 0 signal 0 success=true
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 0 success in 1 seconds
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 0 output
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 wait for last 0 children to finish
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 status: 1 done, 0 running, 1 todo
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 start level 1
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 status: 1 done, 0 running, 1 todo
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 1 job_task qr1hi-ot0gb-w8ujbnulxjaamxf
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 1 child 4984 started on compute13.1
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 status: 1 done, 1 running, 0 todo
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 1 child 4984 on compute13.1 exit 0 signal 0 success=true
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 1 success in 110 seconds
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 1 output bedd6ff56b3ae9f90d873b1fcb72f9a3+91
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 wait for last 0 children to finish
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 status: 2 done, 0 running, 0 todo
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 release job allocation
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 Freeze not implemented
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 collate
-Tue Dec 17 19:04:10 2013 salloc: Job allocation 1251 has been revoked.
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 output bedd6ff56b3ae9f90d873b1fcb72f9a3+91
-Tue Dec 17 19:04:11 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 finish
-Tue Dec 17 19:04:12 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 log manifest is 1e77aaceee2df499e14dc5dde5c3d328+91
</code></pre>
</notextile>
diff --git a/doc/user/topics/tutorial-job1.html.textile.liquid b/doc/user/topics/tutorial-job1.html.textile.liquid
index f69381e..cf09804 100644
--- a/doc/user/topics/tutorial-job1.html.textile.liquid
+++ b/doc/user/topics/tutorial-job1.html.textile.liquid
@@ -76,8 +76,7 @@ Use @arv job create@ to actually submit the job. It should print out a JSON obj
"tasks_summary":{},
"dependencies":[
"c1bad4b39ca5a924e481008009d94e32+210"
- ],
- "log_stream_href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss/log_tail_follow"
+ ]
}
</code></pre>
</notextile>
@@ -91,12 +90,6 @@ h2. Monitor job progress
Go to the "Workbench dashboard":https://{{site.arvados_workbench_host}} and visit *Activity* %(rarr)→% *Recent jobs*. Your job should be near the top of the table. This table refreshes automatically. When the job has completed successfully, it will show <span class="label label-success">finished</span> in the *Status* column.
-On the command line, you can access log messages while the job runs using @arv job log_tail_follow@:
-
-notextile. <pre><code>~$ <span class="userinput">arv job log_tail_follow --uuid qr1hi-8i9sb-xxxxxxxxxxxxxxx</span></code></pre>
-
-This will print out the last several lines of the log for that job.
-
h2. Inspect the job output
On the "Workbench dashboard":https://{{site.arvados_workbench_host}}, look for the *Output* column of the *Recent jobs* table. Click on the link under *Output* for your job to go to the files page with the job output. The files page lists all the files that were output by the job. Click on the link under the *file* column to view a file, or click on the download icon <span class="glyphicon glyphicon-download-alt"></span> to download the output file.
@@ -142,8 +135,7 @@ On the command line, you can use @arv job get@ to access a JSON object describin
},
"dependencies":[
"c1bad4b39ca5a924e481008009d94e32+210"
- ],
- "log_stream_href":null
+ ]
}
</code></pre>
</notextile>
diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index ee56301..9043ac4 100644
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -117,51 +117,6 @@ class Arvados::V1::JobsController < ApplicationController
@job.reload
end
end
- @redis = Redis.new(:timeout => 0)
- if @redis.exists @job.uuid
- # A log buffer exists. Start by showing the last few KB.
- @redis.
- getrange(@job.uuid, 0 - [@opts[:buffer_size], 1].max, -1).
- sub(/^[^\n]*\n?/, '').
- split("\n").
- each do |line|
- yield "#{line}\n"
- end
- end
- # TODO: avoid missing log entries between getrange() above and
- # subscribe() below.
- @redis.subscribe(@job.uuid) do |event|
- event.message do |channel, msg|
- if msg == "end"
- @redis.unsubscribe @job.uuid
- else
- yield "#{msg}\n"
- end
- end
- end
- end
- end
-
- def self._log_tail_follow_requires_parameters
- {
- buffer_size: {type: 'integer', required: false, default: 2**13}
- }
- end
- def log_tail_follow
- if !@object.andand.uuid
- return render_not_found
- end
- if client_accepts_plain_text_stream
- self.response.headers['Last-Modified'] = Time.now.ctime.to_s
- self.response_body = LogStreamer.new @object, {
- buffer_size: (params[:buffer_size].to_i rescue 2**13)
- }
- else
- render json: {
- href: url_for(uuid: @object.uuid),
- comment: ('To retrieve the log stream as plain text, ' +
- 'use a request header like "Accept: text/plain"')
- }
end
end
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index fbc5640..7bca409 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -35,8 +35,6 @@ class Job < ArvadosModel
t.add :runtime_constraints
t.add :tasks_summary
t.add :dependencies
- t.add :log_stream_href
- t.add :log_buffer
t.add :nondeterministic
t.add :repository
end
@@ -47,12 +45,6 @@ class Job < ArvadosModel
running: false)
end
- def log_stream_href
- unless self.finished_at
- "#{current_api_base}/#{self.class.to_s.pluralize.underscore}/#{self.uuid}/log_tail_follow"
- end
- end
-
def self.queue
self.where('started_at is ? and is_locked_by_uuid is ? and cancelled_at is ? and success is ?',
nil, nil, nil, nil).
@@ -187,15 +179,4 @@ class Job < ArvadosModel
end
end
end
-
- def log_buffer
- begin
- @@redis ||= Redis.new(:timeout => 0)
- if @@redis.exists uuid
- @@redis.getrange(uuid, 0 - 2**10, -1)
- end
- rescue Redis::CannotConnectError
- return '(not available)'
- end
- end
end
diff --git a/services/api/config/routes.rb b/services/api/config/routes.rb
index 49d78e4..e4d2975 100644
--- a/services/api/config/routes.rb
+++ b/services/api/config/routes.rb
@@ -21,7 +21,6 @@ Server::Application.routes.draw do
resources :job_tasks
resources :jobs do
get 'queue', on: :collection
- get 'log_tail_follow', on: :member
post 'cancel', on: :member
end
resources :keep_disks do
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list