[ARVADOS] updated: fbf2382d1455a3663e04e07931bc39b1ab088172
git at public.curoverse.com
git at public.curoverse.com
Fri Apr 24 18:11:54 EDT 2015
Summary of changes:
apps/workbench/app/controllers/jobs_controller.rb | 2 -
apps/workbench/app/helpers/jobs_helper.rb | 22 --------
apps/workbench/app/models/pipeline_instance.rb | 49 ++++++++++++++++
.../app/views/application/_content.html.erb | 12 ++--
.../views/pipeline_instances/_show_log.html.erb | 54 ++++++++++++++----
.../test/integration/anonymous_access_test.rb | 29 +++++-----
.../test/integration/pipeline_instances_test.rb | 41 ++++++++++++++
apps/workbench/test/unit/pipeline_instance_test.rb | 66 +++++++++++++++++++++-
services/api/Rakefile | 21 +++++++
services/api/config/application.default.yml | 3 -
services/api/lib/tasks/test_tasks.rake | 6 ++
services/api/test/fixtures/jobs.yml | 18 ++++++
services/api/test/fixtures/logs.yml | 38 +++++++++++++
services/api/test/fixtures/pipeline_instances.yml | 42 +++++++++++++-
.../api/test/tasks/delete_old_job_logs_test.rb | 50 ++++++++++++++++
15 files changed, 390 insertions(+), 63 deletions(-)
delete mode 100644 apps/workbench/app/helpers/jobs_helper.rb
create mode 100644 services/api/lib/tasks/test_tasks.rake
create mode 100644 services/api/test/tasks/delete_old_job_logs_test.rb
via fbf2382d1455a3663e04e07931bc39b1ab088172 (commit)
via 5374510c9b4edce3161b7df345843a294a5baf06 (commit)
via 1aee2f3722336df3097b3188ed221873aa3a4238 (commit)
from 0bef99f422b367c5a36dc0e14bab0a1a64337088 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fbf2382d1455a3663e04e07931bc39b1ab088172
Author: Brett Smith <brett at curoverse.com>
Date: Fri Apr 24 17:13:31 2015 -0400
5554: Workbench links finished job logs from pipeline instance log tab.
This is an affordance to help users find logs after they're removed
from the API server's logs table. The display is not very great.
We're punting improvements until Workbench generally has better
capabilities to display large logs.
The layout change necessitated updates to the anonymous access tests.
While I was in there, I strengthened the assertions by putting
assertions that check a particular element has loaded before asserting
the nonexistence of other elements. This revealed the JavaScript
issue mentioned in the new comment: the old version of the test was
doing a single `assert_no_text` after clicking a job's log tab, which
only confirmed that we navigated away. We already test that the log
tab exists and is not disabled in this case, so I consider that
sufficient for now.
diff --git a/apps/workbench/app/controllers/jobs_controller.rb b/apps/workbench/app/controllers/jobs_controller.rb
index 7edf8cc..3984177 100644
--- a/apps/workbench/app/controllers/jobs_controller.rb
+++ b/apps/workbench/app/controllers/jobs_controller.rb
@@ -4,8 +4,6 @@ class JobsController < ApplicationController
'show' == ctrl.action_name
}
- include JobsHelper
-
def generate_provenance(jobs)
return if params['tab_pane'] != "Provenance"
diff --git a/apps/workbench/app/helpers/jobs_helper.rb b/apps/workbench/app/helpers/jobs_helper.rb
deleted file mode 100644
index 889dd23..0000000
--- a/apps/workbench/app/helpers/jobs_helper.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-module JobsHelper
- def stderr_log_history(job_uuids, limit=2000)
- results = []
-
- log_history = Log.where(event_type: 'stderr',
- object_uuid: job_uuids).limit(limit).order('id DESC')
- if !log_history.results.empty?
- reversed_results = log_history.results.reverse
- reversed_results.each do |entry|
- if entry.andand.properties
- properties = entry.properties
- text = properties[:text]
- if text
- results = results.concat text.split("\n")
- end
- end
- end
- end
- return results
- end
-
-end
diff --git a/apps/workbench/app/models/pipeline_instance.rb b/apps/workbench/app/models/pipeline_instance.rb
index f575e20..01bd64d 100644
--- a/apps/workbench/app/models/pipeline_instance.rb
+++ b/apps/workbench/app/models/pipeline_instance.rb
@@ -81,4 +81,53 @@ class PipelineInstance < ArvadosBase
def textile_attributes
[ 'description' ]
end
+
+ def job_uuids
+ components_map { |cspec| cspec[:job][:uuid] rescue nil }
+ end
+
+ def job_log_ids
+ components_map { |cspec| cspec[:job][:log] rescue nil }
+ end
+
+ def stderr_log_object_uuids
+ result = job_uuids.values.compact
+ result << uuid
+ end
+
+ def stderr_log_query(limit=nil)
+ query = Log.
+ where(event_type: "stderr",
+ object_uuid: stderr_log_object_uuids).
+ order("id DESC")
+ unless limit.nil?
+ query = query.limit(limit)
+ end
+ query
+ end
+
+ def stderr_log_lines(limit=2000)
+ stderr_log_query(limit).results.reverse.
+ flat_map { |log| log.properties[:text].split("\n") rescue [] }
+ end
+
+ def has_readable_logs?
+ log_pdhs, log_uuids =
+ job_log_ids.values.compact.partition { |s| s =~ /^[0-9a-f]{32}\b/ }
+ if log_pdhs.any? and
+ Collection.where(portable_data_hash: log_pdhs).limit(1).results.any?
+ true
+ elsif log_uuids.any? and
+ Collection.where(uuid: log_uuids).limit(1).results.any?
+ true
+ else
+ stderr_log_query(1).results.any?
+ end
+ end
+
+ private
+
+ def components_map
+ Hash[components.map { |cname, cspec| [cname, yield(cspec)] }]
+ end
end
diff --git a/apps/workbench/app/views/application/_content.html.erb b/apps/workbench/app/views/application/_content.html.erb
index 5e6211f..9441a46 100644
--- a/apps/workbench/app/views/application/_content.html.erb
+++ b/apps/workbench/app/views/application/_content.html.erb
@@ -20,13 +20,11 @@
link_disabled = "disabled"
end
end
- elsif controller.model_class.to_s == 'PipelineInstance'
- log_uuids = [@object.uuid] + pipeline_jobs(@object).collect{|x|x[:job].andand[:uuid]}.compact
- if stderr_log_history(log_uuids, 1).empty?
- data_toggle = "disabled"
- tab_tooltip = "Log data is not available"
- link_disabled = "disabled"
- end
+ elsif (controller.model_class.to_s == 'PipelineInstance' and
+ !@object.has_readable_logs?)
+ data_toggle = "disabled"
+ tab_tooltip = "Log data is not available"
+ link_disabled = "disabled"
end
end
%>
diff --git a/apps/workbench/app/views/pipeline_instances/_show_log.html.erb b/apps/workbench/app/views/pipeline_instances/_show_log.html.erb
index bb756a0..1f1e62a 100644
--- a/apps/workbench/app/views/pipeline_instances/_show_log.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/_show_log.html.erb
@@ -1,12 +1,42 @@
-<% log_uuids = [@object.uuid] + pipeline_jobs(@object).collect{|x|x[:job].andand[:uuid]}.compact %>
-<% log_history = stderr_log_history(log_uuids) %>
-<div id="event_log_div"
- class="arv-log-event-listener arv-log-event-handler-append-logs arv-log-event-subscribe-to-pipeline-job-uuids arv-job-log-window"
- data-object-uuids="<%= log_uuids.join(' ') %>"
- ><%= log_history.join("\n") %></div>
-
-<%# Applying a long throttle suppresses the auto-refresh of this
- partial that would normally be triggered by arv-log-event. %>
-<div class="arv-log-refresh-control"
- data-load-throttle="86486400000" <%# 1001 nights %>
- ></div>
+<% log_ids = @object.job_log_ids
+ still_logging, done_logging = log_ids.keys.partition { |k| log_ids[k].nil? }
+%>
+
+<% unless done_logging.empty? %>
+ <table class="topalign table table-condensed table-fixedlayout">
+ <colgroup>
+ <col width="40%" />
+ <col width="60%" />
+ </colgroup>
+ <thead>
+ <tr>
+ <th>finished component</th>
+ <th>job log</th>
+ </tr>
+ </thead>
+ <tbody>
+ <% done_logging.each do |cname| %>
+ <tr>
+ <td><%= cname %></td>
+ <td><%= link_to("Log for #{cname}",
+ {controller: "collections", action: "show", id: log_ids[cname]})
+ %></td>
+ </tr>
+ <% end %>
+ </tbody>
+ </table>
+<% end %>
+
+<% unless still_logging.empty? %>
+ <div id="event_log_div"
+ class="arv-log-event-listener arv-log-event-handler-append-logs arv-log-event-subscribe-to-pipeline-job-uuids arv-job-log-window"
+ data-object-uuids="<%= @object.stderr_log_object_uuids.join(' ') %>"
+ ><%= @object.stderr_log_lines.join("\n") %></div>
+
+ <%# Applying a long throttle suppresses the auto-refresh of this
+ partial that would normally be triggered by arv-log-event. %>
+ <div class="arv-log-refresh-control"
+ data-load-throttle="86486400000" <%# 1001 nights %>
+ ></div>
+<% end %>
+
diff --git a/apps/workbench/test/integration/anonymous_access_test.rb b/apps/workbench/test/integration/anonymous_access_test.rb
index 00ec219..18c5b7d 100644
--- a/apps/workbench/test/integration/anonymous_access_test.rb
+++ b/apps/workbench/test/integration/anonymous_access_test.rb
@@ -211,9 +211,11 @@ class AnonymousAccessTest < ActionDispatch::IntegrationTest
if pipeline_page
object = api_fixture('pipeline_instances')[fixture]
page = "/pipeline_instances/#{object['uuid']}"
+ expect_log_text = "Log for foo"
else # job
object = api_fixture('jobs')[fixture]
page = "/jobs/#{object['uuid']}"
+ expect_log_text = "stderr crunchstat"
end
if user
@@ -226,30 +228,29 @@ class AnonymousAccessTest < ActionDispatch::IntegrationTest
click_link 'foo' if pipeline_page
if objects_readable
+ assert_selector 'a[href="#Log"]', text: 'Log'
+ assert_no_selector 'a[data-toggle="disabled"]', text: 'Log'
+ assert_no_text 'Output data not available'
if pipeline_page
assert_text 'This pipeline was created from'
assert_selector 'a', text: object['components']['foo']['job']['uuid']
+ # We'd like to test the Log tab on job pages too, but we can't right
+ # now because Poltergeist 1.x doesn't support JavaScript's
+ # Function.prototype.bind, which is used by job_log_graph.js.
+ click_link "Log"
+ assert_text expect_log_text
end
- assert_no_text 'Output data not available'
- assert_selector 'a[href="#Log"]', text: 'Log'
- assert_no_selector 'a[data-toggle="disabled"]', text: 'Log'
else
+ assert_selector 'a[data-toggle="disabled"]', text: 'Log'
+ assert_text 'Output data not available'
+ assert_text object['job']
if pipeline_page
assert_no_text 'This pipeline was created from' # template is not readable
assert_no_selector 'a', text: object['components']['foo']['job']['uuid']
end
+ click_link "Log"
assert_text 'Output data not available'
- assert_text object['job']
- assert_selector 'a[data-toggle="disabled"]', text: 'Log'
- end
-
- click_link 'Log'
- if objects_readable
- assert_no_text 'foo' # should be in Log tab
- assert_text 'stderr crunchstat' if pipeline_page
- else
- assert_text 'foo' # Log tab disabled and hence still in first tab
- assert_no_text 'stderr crunchstat' # log line shouldn't be seen
+ assert_no_text expect_log_text
end
end
end
diff --git a/apps/workbench/test/integration/pipeline_instances_test.rb b/apps/workbench/test/integration/pipeline_instances_test.rb
index f291674..da8f439 100644
--- a/apps/workbench/test/integration/pipeline_instances_test.rb
+++ b/apps/workbench/test/integration/pipeline_instances_test.rb
@@ -541,4 +541,45 @@ class PipelineInstancesTest < ActionDispatch::IntegrationTest
visit page_with_token 'active', '/pipeline_instances/' + pi['uuid']
assert_text 'Queued for '
end
+
+ test "job logs linked for running pipeline" do
+ pi = api_fixture("pipeline_instances", "running_pipeline_with_complete_job")
+ visit(page_with_token("active", "/pipeline_instances/#{pi['uuid']}"))
+ click_on "Log"
+ within "#Log" do
+ assert_text "Log for previous"
+ log_link = find("a", text: "Log for previous")
+ assert_includes(log_link[:href],
+ pi["components"]["previous"]["job"]["log"])
+ assert_selector "#event_log_div"
+ end
+ end
+
+ test "job logs linked for complete pipeline" do
+ pi = api_fixture("pipeline_instances", "complete_pipeline_with_two_jobs")
+ visit(page_with_token("active", "/pipeline_instances/#{pi['uuid']}"))
+ click_on "Log"
+ within "#Log" do
+ assert_text "Log for previous"
+ pi["components"].each do |cname, cspec|
+ log_link = find("a", text: "Log for #{cname}")
+ assert_includes(log_link[:href], cspec["job"]["log"])
+ end
+ assert_no_selector "#event_log_div"
+ end
+ end
+
+ test "job logs linked for failed pipeline" do
+ pi = api_fixture("pipeline_instances", "failed_pipeline_with_two_jobs")
+ visit(page_with_token("active", "/pipeline_instances/#{pi['uuid']}"))
+ click_on "Log"
+ within "#Log" do
+ assert_text "Log for previous"
+ pi["components"].each do |cname, cspec|
+ log_link = find("a", text: "Log for #{cname}")
+ assert_includes(log_link[:href], cspec["job"]["log"])
+ end
+ assert_no_selector "#event_log_div"
+ end
+ end
end
diff --git a/apps/workbench/test/unit/pipeline_instance_test.rb b/apps/workbench/test/unit/pipeline_instance_test.rb
index 4cad6e6..747cfc1 100644
--- a/apps/workbench/test/unit/pipeline_instance_test.rb
+++ b/apps/workbench/test/unit/pipeline_instance_test.rb
@@ -1,9 +1,13 @@
require 'test_helper'
class PipelineInstanceTest < ActiveSupport::TestCase
- def attribute_editable_for?(token_name, pi_name, attr_name, ever=nil)
+ def find_pi_with(token_name, pi_name)
use_token token_name
- find_fixture(PipelineInstance, pi_name).attribute_editable?(attr_name, ever)
+ find_fixture(PipelineInstance, pi_name)
+ end
+
+ def attribute_editable_for?(token_name, pi_name, attr_name, ever=nil)
+ find_pi_with(token_name, pi_name).attribute_editable?(attr_name, ever)
end
test "admin can edit name" do
@@ -46,4 +50,62 @@ class PipelineInstanceTest < ActiveSupport::TestCase
"components"),
"components not editable on new pipeline")
end
+
+ test "job_logs for partially complete pipeline" do
+ log_uuid = api_fixture("collections", "real_log_collection", "uuid")
+ pi = find_pi_with(:active, "running_pipeline_with_complete_job")
+ assert_equal({previous: log_uuid, running: nil}, pi.job_log_ids)
+ end
+
+ test "job_logs for complete pipeline" do
+ log_uuid = api_fixture("collections", "real_log_collection", "uuid")
+ pi = find_pi_with(:active, "complete_pipeline_with_two_jobs")
+ assert_equal({ancient: log_uuid, previous: log_uuid}, pi.job_log_ids)
+ end
+
+ test "job_logs for malformed pipeline" do
+ pi = find_pi_with(:active, "components_is_jobspec")
+ assert_empty(pi.job_log_ids.select { |_, log| not log.nil? })
+ end
+
+ def check_stderr_logs(token_name, pi_name, log_name)
+ pi = find_pi_with(token_name, pi_name)
+ actual_logs = pi.stderr_log_lines
+ expected_text = api_fixture("logs", log_name, "properties", "text")
+ expected_text.each_line do |log_line|
+ assert_includes(actual_logs, log_line.chomp)
+ end
+ end
+
+ test "stderr_logs for running pipeline" do
+ check_stderr_logs(:active,
+ "pipeline_in_publicly_accessible_project",
+ "log_line_for_pipeline_in_publicly_accessible_project")
+ end
+
+ test "stderr_logs for job in complete pipeline" do
+ check_stderr_logs(:active,
+ "failed_pipeline_with_two_jobs",
+ "crunchstat_for_previous_job")
+ end
+
+ test "has_readable_logs? for unrun pipeline" do
+ pi = find_pi_with(:active, "new_pipeline")
+ refute(pi.has_readable_logs?)
+ end
+
+ test "has_readable_logs? for running pipeline" do
+ pi = find_pi_with(:active, "running_pipeline_with_complete_job")
+ assert(pi.has_readable_logs?)
+ end
+
+ test "has_readable_logs? for complete pipeline" do
+ pi = find_pi_with(:active, "pipeline_in_publicly_accessible_project_but_other_objects_elsewhere")
+ assert(pi.has_readable_logs?)
+ end
+
+ test "has_readable_logs? for complete pipeline when jobs unreadable" do
+ pi = find_pi_with(:anonymous, "pipeline_in_publicly_accessible_project_but_other_objects_elsewhere")
+ refute(pi.has_readable_logs?)
+ end
end
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 2c69325..e7dbf29 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -279,7 +279,4 @@ common:
# When you run the db:delete_old_job_logs task, it will find jobs that
# have been finished for at least this many seconds, and delete their
# stderr logs from the logs table.
- # If a pipeline runs longer than this, client applications like
- # Workbench will lose the ability to display logs from jobs in the
- # pipeline that finished this long ago.
clean_job_log_rows_after: <%= 30.days %>
diff --git a/services/api/test/fixtures/pipeline_instances.yml b/services/api/test/fixtures/pipeline_instances.yml
index 7e9b6cd..41a7fc9 100644
--- a/services/api/test/fixtures/pipeline_instances.yml
+++ b/services/api/test/fixtures/pipeline_instances.yml
@@ -371,6 +371,47 @@ pipeline_in_running_state:
uuid: zzzzz-8i9sb-pshmckwoma9plh7
script_version: master
+running_pipeline_with_complete_job:
+ uuid: zzzzz-d1hrv-partdonepipelin
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: RunningOnServer
+ components:
+ previous:
+ job:
+ uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
+ log: zzzzz-4zz18-op4e2lbej01tcvu
+ running:
+ job:
+ uuid: zzzzz-8i9sb-pshmckwoma9plh7
+
+complete_pipeline_with_two_jobs:
+ uuid: zzzzz-d1hrv-twodonepipeline
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Complete
+ components:
+ ancient:
+ job:
+ uuid: zzzzz-8i9sb-ahd7cie8jah9qui
+ log: zzzzz-4zz18-op4e2lbej01tcvu
+ previous:
+ job:
+ uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
+ log: zzzzz-4zz18-op4e2lbej01tcvu
+
+failed_pipeline_with_two_jobs:
+ uuid: zzzzz-d1hrv-twofailpipeline
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Failed
+ components:
+ ancient:
+ job:
+ uuid: zzzzz-8i9sb-ahd7cie8jah9qui
+ log: zzzzz-4zz18-op4e2lbej01tcvu
+ previous:
+ job:
+ uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
+ log: zzzzz-4zz18-op4e2lbej01tcvu
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
commit 5374510c9b4edce3161b7df345843a294a5baf06
Author: Brett Smith <brett at curoverse.com>
Date: Thu Apr 23 16:52:07 2015 -0400
5554: Clean up erroneous fixture pipeline state.
diff --git a/services/api/test/fixtures/pipeline_instances.yml b/services/api/test/fixtures/pipeline_instances.yml
index 41eb044..7e9b6cd 100644
--- a/services/api/test/fixtures/pipeline_instances.yml
+++ b/services/api/test/fixtures/pipeline_instances.yml
@@ -357,7 +357,6 @@ new_pipeline_in_publicly_accessible_project_with_dataclass_file_and_other_object
pipeline_in_running_state:
name: running_with_job
- state: Ready
uuid: zzzzz-d1hrv-runningpipeline
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: <%= 3.1.minute.ago.to_s(:db) %>
commit 1aee2f3722336df3097b3188ed221873aa3a4238
Author: Brett Smith <brett at curoverse.com>
Date: Thu Apr 23 16:32:25 2015 -0400
5554: Fixup: Test the Rake task.
diff --git a/services/api/Rakefile b/services/api/Rakefile
index 223f5ca..22b2531 100644
--- a/services/api/Rakefile
+++ b/services/api/Rakefile
@@ -11,3 +11,24 @@ rescue
end
Server::Application.load_tasks
+
+namespace :test do
+ task(:run).clear
+ # Copied from the definition in Rails 3.2.
+ # This may need to be updated if we upgrade Rails.
+ task :run do
+ errors = %w(test:units test:functionals test:integration test:tasks).collect do |task|
+ begin
+ Rake::Task[task].invoke
+ nil
+ rescue => e
+ { :task => task, :exception => e }
+ end
+ end.compact
+
+ if errors.any?
+ puts errors.map { |e| "Errors running #{e[:task]}! #{e[:exception].inspect}" }.join("\n")
+ abort
+ end
+ end
+end
diff --git a/services/api/lib/tasks/test_tasks.rake b/services/api/lib/tasks/test_tasks.rake
new file mode 100644
index 0000000..27bf232
--- /dev/null
+++ b/services/api/lib/tasks/test_tasks.rake
@@ -0,0 +1,6 @@
+namespace :test do
+ new_task = Rake::TestTask.new(tasks: "test:prepare") do |t|
+ t.libs << "test"
+ t.pattern = "test/tasks/**/*_test.rb"
+ end
+end
diff --git a/services/api/test/fixtures/jobs.yml b/services/api/test/fixtures/jobs.yml
index 23b32ef..aa16134 100644
--- a/services/api/test/fixtures/jobs.yml
+++ b/services/api/test/fixtures/jobs.yml
@@ -163,6 +163,7 @@ runningbarbaz:
previous_job_run:
uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
created_at: <%= 14.minute.ago.to_s(:db) %>
+ finished_at: <%= 13.minutes.ago.to_s(:db) %>
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
repository: active/foo
script: hash
@@ -171,6 +172,23 @@ previous_job_run:
input: fa7aeb5140e2848d39b416daeef4ffc5+45
an_integer: "1"
success: true
+ log: d41d8cd98f00b204e9800998ecf8427e+0
+ output: ea10d51bcf88862dbcc36eb292017dfd+45
+ state: Complete
+
+previous_ancient_job_run:
+ uuid: zzzzz-8i9sb-ahd7cie8jah9qui
+ created_at: <%= 366.days.ago.to_s(:db) %>
+ finished_at: <%= 365.days.ago.to_s(:db) %>
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ repository: active/foo
+ script: hash
+ script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
+ script_parameters:
+ input: fa7aeb5140e2848d39b416daeef4ffc5+45
+ an_integer: "2"
+ success: true
+ log: d41d8cd98f00b204e9800998ecf8427e+0
output: ea10d51bcf88862dbcc36eb292017dfd+45
state: Complete
diff --git a/services/api/test/fixtures/logs.yml b/services/api/test/fixtures/logs.yml
index aea7980..9179e6d 100644
--- a/services/api/test/fixtures/logs.yml
+++ b/services/api/test/fixtures/logs.yml
@@ -101,3 +101,41 @@ log_line_for_pipeline_in_publicly_accessible_project_but_other_objects_elsewhere
updated_at: 2014-11-07 23:33:42.347455000 Z
modified_at: 2014-11-07 23:33:42.347455000 Z
object_owner_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+
+crunchstat_for_previous_job:
+ id: 10
+ uuid: zzzzz-57u5n-eir3aesha3kaene
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ object_uuid: zzzzz-8i9sb-cjs4pklxxjykqqq
+ event_at: 2014-11-07 23:33:42.347455000 Z
+ event_type: stderr
+ summary: ~
+ properties:
+ text: '2014-11-07_23:33:41 zzzzz-8i9sb-cjs4pklxxjykqqq 11592 1 stderr crunchstat:
+ cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+ 0.9900 sys'
+ created_at: 2014-11-07 23:33:42.351913000 Z
+ updated_at: 2014-11-07 23:33:42.347455000 Z
+ modified_at: 2014-11-07 23:33:42.347455000 Z
+ object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+crunchstat_for_ancient_job:
+ id: 11
+ uuid: zzzzz-57u5n-ixioph7ieb5ung8
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ object_uuid: zzzzz-8i9sb-ahd7cie8jah9qui
+ event_at: 2013-11-07 23:33:42.347455000 Z
+ event_type: stderr
+ summary: ~
+ properties:
+ text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+ cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+ 0.9900 sys'
+ created_at: 2013-11-07 23:33:42.351913000 Z
+ updated_at: 2013-11-07 23:33:42.347455000 Z
+ modified_at: 2013-11-07 23:33:42.347455000 Z
+ object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
diff --git a/services/api/test/tasks/delete_old_job_logs_test.rb b/services/api/test/tasks/delete_old_job_logs_test.rb
new file mode 100644
index 0000000..b922fb3
--- /dev/null
+++ b/services/api/test/tasks/delete_old_job_logs_test.rb
@@ -0,0 +1,50 @@
+require 'test_helper'
+require 'rake'
+
+Rake.application.rake_require "tasks/delete_old_job_logs"
+Rake::Task.define_task(:environment)
+
+class DeleteOldJobLogsTaskTest < ActiveSupport::TestCase
+ TASK_NAME = "db:delete_old_job_logs"
+
+ def log_uuids(*fixture_names)
+ fixture_names.map { |name| logs(name).uuid }
+ end
+
+ def run_with_expiry(clean_after)
+ Rails.configuration.clean_job_log_rows_after = clean_after
+ Rake::Task[TASK_NAME].reenable
+ Rake.application.invoke_task TASK_NAME
+ end
+
+ def job_stderr_logs
+ Log.where("object_uuid LIKE :pattern AND event_type = :etype",
+ pattern: "_____-8i9sb-_______________",
+ etype: "stderr")
+ end
+
+ def check_existence(test_method, fixture_uuids)
+ uuids_now = job_stderr_logs.map(&:uuid)
+ fixture_uuids.each do |expect_uuid|
+ send(test_method, uuids_now, expect_uuid)
+ end
+ end
+
+ test "delete all logs" do
+ uuids_to_keep = log_uuids(:crunchstat_for_running_job)
+ uuids_to_clean = log_uuids(:crunchstat_for_previous_job,
+ :crunchstat_for_ancient_job)
+ run_with_expiry(1)
+ check_existence(:assert_includes, uuids_to_keep)
+ check_existence(:refute_includes, uuids_to_clean)
+ end
+
+ test "delete only old logs" do
+ uuids_to_keep = log_uuids(:crunchstat_for_running_job,
+ :crunchstat_for_previous_job)
+ uuids_to_clean = log_uuids(:crunchstat_for_ancient_job)
+ run_with_expiry(360.days)
+ check_existence(:assert_includes, uuids_to_keep)
+ check_existence(:refute_includes, uuids_to_clean)
+ end
+end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list