[ARVADOS] updated: b402e6a66e569167c0e990a4fdfc2843df76afd8

git at public.curoverse.com git at public.curoverse.com
Fri May 30 16:22:08 EDT 2014


Summary of changes:
 ...rial-gatk-variantfiltration.html.textile.liquid | 42 +----------
 doc/user/topics/tutorial-job1.html.textile.liquid  | 12 +---
 sdk/python/arvados/collection.py                   | 21 +++++-
 sdk/python/arvados/commands/put.py                 |  4 +-
 sdk/python/test_cmdline.py                         | 81 ++++++++++++++++++++++
 sdk/python/tests/test_keep_client.py               | 20 ++++--
 .../app/controllers/arvados/v1/jobs_controller.rb  | 45 ------------
 services/api/app/models/job.rb                     | 19 -----
 services/api/config/application.yml.example        | 10 +--
 services/api/config/routes.rb                      |  1 -
 .../arvados/v1/collections_controller_test.rb      | 14 ++++
 11 files changed, 140 insertions(+), 129 deletions(-)
 create mode 100644 sdk/python/test_cmdline.py

       via  b402e6a66e569167c0e990a4fdfc2843df76afd8 (commit)
       via  070e2e8e80cfc88414dd2595cae5bfddce1cd94f (commit)
       via  b6ea1fe3bf38bf28823c80b3aef98239a1c0311b (commit)
       via  c1e7255a85dfc2807ba78e1cf9d109d896c80b42 (commit)
       via  39b2ed3024860262747fd6622433e0bf7ba39530 (commit)
       via  5b15dc0308bd3ddcc3c50828269888b5811d0007 (commit)
       via  3107d80e87c4fd8a4c6f6e06d9a22537bc1f19b0 (commit)
       via  b2812e9406253ded204ac853fb47437e8118bcdd (commit)
       via  9d4f6f0284011fbc7a8edd203d1fab93a9a61321 (commit)
       via  30b6c5a0f8dc91c76ca30ba4b5263b7eab858bdb (commit)
       via  46460c9615f07184ecb9917e4c948635712859da (commit)
      from  ff3686b372cd598ee990acd990af2b04daf91ada (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit b402e6a66e569167c0e990a4fdfc2843df76afd8
Merge: 070e2e8 b6ea1fe
Author: radhika <radhika at curoverse.com>
Date:   Fri May 30 16:21:49 2014 -0400

    Merge branch 'master' into 2895-no-more-redis


commit 070e2e8e80cfc88414dd2595cae5bfddce1cd94f
Author: radhika <radhika at curoverse.com>
Date:   Fri May 30 16:20:19 2014 -0400

    2895: remove jobs.log_tail_follow api since we are now using websockets instead of redis

diff --git a/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid b/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
index 0248325..ea608b4 100644
--- a/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
+++ b/doc/user/topics/tutorial-gatk-variantfiltration.html.textile.liquid
@@ -154,48 +154,8 @@ Now start a job:
   "5ee633fe2569d2a42dd81b07490d5d13+82",
   "c905c8d8443a9c44274d98b7c6cfaa32+94",
   "d237a90bae3870b3b033aea1e99de4a9+10820"
- ],
- "log_stream_href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-n9k7qyp7bs5b9d4/log_tail_follow"
+ ]
 }
-~$ <span class="userinput">arv job log_tail_follow --uuid qr1hi-8i9sb-n9k7qyp7bs5b9d4</span>
-Tue Dec 17 19:02:16 2013 salloc: Granted job allocation 1251
-Tue Dec 17 19:02:17 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  check slurm allocation
-Tue Dec 17 19:02:17 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  node compute13 - 8 slots
-Tue Dec 17 19:02:17 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  start
-Tue Dec 17 19:02:17 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  Install revision 76588bfc57f33ea1b36b82ca7187f465b73b4ca4
-Tue Dec 17 19:02:18 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  Clean-work-dir exited 0
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  Install exited 0
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  script GATK2-VariantFiltration
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  script_version 76588bfc57f33ea1b36b82ca7187f465b73b4ca4
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  script_parameters {"input":"5ee633fe2569d2a42dd81b07490d5d13+82","gatk_bundle":"d237a90bae3870b3b033aea1e99de4a9+10820","gatk_binary_tarball":"c905c8d8443a9c44274d98b7c6cfaa32+94"}
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  runtime_constraints {"max_tasks_per_node":0}
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  start level 0
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  status: 0 done, 0 running, 1 todo
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 0 job_task qr1hi-ot0gb-d3sjxerucfbvyev
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 0 child 4946 started on compute13.1
-Tue Dec 17 19:02:19 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  status: 0 done, 1 running, 0 todo
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 0 child 4946 on compute13.1 exit 0 signal 0 success=true
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 0 success in 1 seconds
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 0 output
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  wait for last 0 children to finish
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  status: 1 done, 0 running, 1 todo
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  start level 1
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  status: 1 done, 0 running, 1 todo
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 1 job_task qr1hi-ot0gb-w8ujbnulxjaamxf
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 1 child 4984 started on compute13.1
-Tue Dec 17 19:02:20 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  status: 1 done, 1 running, 0 todo
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 1 child 4984 on compute13.1 exit 0 signal 0 success=true
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 1 success in 110 seconds
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867 1 output bedd6ff56b3ae9f90d873b1fcb72f9a3+91
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  wait for last 0 children to finish
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  status: 2 done, 0 running, 0 todo
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  release job allocation
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  Freeze not implemented
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  collate
-Tue Dec 17 19:04:10 2013 salloc: Job allocation 1251 has been revoked.
-Tue Dec 17 19:04:10 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  output bedd6ff56b3ae9f90d873b1fcb72f9a3+91
-Tue Dec 17 19:04:11 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  finish
-Tue Dec 17 19:04:12 2013 qr1hi-8i9sb-n9k7qyp7bs5b9d4 4867  log manifest is 1e77aaceee2df499e14dc5dde5c3d328+91
 </code></pre>
 </notextile>
 
diff --git a/doc/user/topics/tutorial-job1.html.textile.liquid b/doc/user/topics/tutorial-job1.html.textile.liquid
index f69381e..cf09804 100644
--- a/doc/user/topics/tutorial-job1.html.textile.liquid
+++ b/doc/user/topics/tutorial-job1.html.textile.liquid
@@ -76,8 +76,7 @@ Use @arv job create@ to actually submit the job.  It should print out a JSON obj
  "tasks_summary":{},
  "dependencies":[
   "c1bad4b39ca5a924e481008009d94e32+210"
- ],
- "log_stream_href":"https://qr1hi.arvadosapi.com/arvados/v1/jobs/qr1hi-8i9sb-1pm1t02dezhupss/log_tail_follow"
+ ]
 }
 </code></pre>
 </notextile>
@@ -91,12 +90,6 @@ h2. Monitor job progress
 
 Go to the "Workbench dashboard":https://{{site.arvados_workbench_host}} and visit *Activity* %(rarr)→% *Recent jobs*.  Your job should be near the top of the table.  This table refreshes automatically.  When the job has completed successfully, it will show <span class="label label-success">finished</span> in the *Status* column.
 
-On the command line, you can access log messages while the job runs using @arv job log_tail_follow@:
-
-notextile. <pre><code>~$ <span class="userinput">arv job log_tail_follow --uuid qr1hi-8i9sb-xxxxxxxxxxxxxxx</span></code></pre>
-
-This will print out the last several lines of the log for that job.
-
 h2. Inspect the job output
 
 On the "Workbench dashboard":https://{{site.arvados_workbench_host}}, look for the *Output* column of the *Recent jobs* table.  Click on the link under *Output* for your job to go to the files page with the job output.  The files page lists all the files that were output by the job.  Click on the link under the *file* column to view a file, or click on the download icon <span class="glyphicon glyphicon-download-alt"></span> to download the output file.
@@ -142,8 +135,7 @@ On the command line, you can use @arv job get@ to access a JSON object describin
  },
  "dependencies":[
   "c1bad4b39ca5a924e481008009d94e32+210"
- ],
- "log_stream_href":null
+ ]
 }
 </code></pre>
 </notextile>
diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index ee56301..9043ac4 100644
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -117,51 +117,6 @@ class Arvados::V1::JobsController < ApplicationController
           @job.reload
         end
       end
-      @redis = Redis.new(:timeout => 0)
-      if @redis.exists @job.uuid
-        # A log buffer exists. Start by showing the last few KB.
-        @redis.
-          getrange(@job.uuid, 0 - [@opts[:buffer_size], 1].max, -1).
-          sub(/^[^\n]*\n?/, '').
-          split("\n").
-          each do |line|
-          yield "#{line}\n"
-        end
-      end
-      # TODO: avoid missing log entries between getrange() above and
-      # subscribe() below.
-      @redis.subscribe(@job.uuid) do |event|
-        event.message do |channel, msg|
-          if msg == "end"
-            @redis.unsubscribe @job.uuid
-          else
-            yield "#{msg}\n"
-          end
-        end
-      end
-    end
-  end
-
-  def self._log_tail_follow_requires_parameters
-    {
-      buffer_size: {type: 'integer', required: false, default: 2**13}
-    }
-  end
-  def log_tail_follow
-    if !@object.andand.uuid
-      return render_not_found
-    end
-    if client_accepts_plain_text_stream
-      self.response.headers['Last-Modified'] = Time.now.ctime.to_s
-      self.response_body = LogStreamer.new @object, {
-        buffer_size: (params[:buffer_size].to_i rescue 2**13)
-      }
-    else
-      render json: {
-        href: url_for(uuid: @object.uuid),
-        comment: ('To retrieve the log stream as plain text, ' +
-                  'use a request header like "Accept: text/plain"')
-      }
     end
   end
 
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index fbc5640..7bca409 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -35,8 +35,6 @@ class Job < ArvadosModel
     t.add :runtime_constraints
     t.add :tasks_summary
     t.add :dependencies
-    t.add :log_stream_href
-    t.add :log_buffer
     t.add :nondeterministic
     t.add :repository
   end
@@ -47,12 +45,6 @@ class Job < ArvadosModel
                       running: false)
   end
 
-  def log_stream_href
-    unless self.finished_at
-      "#{current_api_base}/#{self.class.to_s.pluralize.underscore}/#{self.uuid}/log_tail_follow"
-    end
-  end
-
   def self.queue
     self.where('started_at is ? and is_locked_by_uuid is ? and cancelled_at is ? and success is ?',
                nil, nil, nil, nil).
@@ -187,15 +179,4 @@ class Job < ArvadosModel
       end
     end
   end
-
-  def log_buffer
-    begin
-      @@redis ||= Redis.new(:timeout => 0)
-      if @@redis.exists uuid
-        @@redis.getrange(uuid, 0 - 2**10, -1)
-      end
-    rescue Redis::CannotConnectError
-      return '(not available)'
-    end
-  end
 end
diff --git a/services/api/config/routes.rb b/services/api/config/routes.rb
index 49d78e4..e4d2975 100644
--- a/services/api/config/routes.rb
+++ b/services/api/config/routes.rb
@@ -21,7 +21,6 @@ Server::Application.routes.draw do
       resources :job_tasks
       resources :jobs do
         get 'queue', on: :collection
-        get 'log_tail_follow', on: :member
         post 'cancel', on: :member
       end
       resources :keep_disks do

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list