[ARVADOS] created: e3de0a09618c8aa46833357e6eacabe15bb68f6f
Git user
git at public.curoverse.com
Thu Sep 15 18:32:14 EDT 2016
at e3de0a09618c8aa46833357e6eacabe15bb68f6f (commit)
commit e3de0a09618c8aa46833357e6eacabe15bb68f6f
Author: radhika <radhika at curoverse.com>
Date: Thu Sep 15 18:30:31 2016 -0400
9514: add rake task to delete logs for old containers whose log is already stored in keep.
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 99d2a10..96e7596 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -220,6 +220,12 @@ common:
# stderr logs from the logs table.
clean_job_log_rows_after: <%= 30.days %>
+ # When you run the db:delete_old_container_logs task, it will find
+ # containers that have been finished for at least this many seconds,
+ # and delete their stdout, stderr, arv-mount, crunch-run, and
+ # crunchstat logs from the logs table.
+ clean_container_log_rows_after: <%= 30.days %>
+
# The maximum number of compute nodes that can be in use simultaneously
# If this limit is reduced, any existing nodes with slot number >= new limit
# will not be counted against the new limit. In other words, the new limit
diff --git a/services/api/lib/tasks/delete_old_container_logs.rake b/services/api/lib/tasks/delete_old_container_logs.rake
new file mode 100644
index 0000000..8c32331
--- /dev/null
+++ b/services/api/lib/tasks/delete_old_container_logs.rake
@@ -0,0 +1,17 @@
+# This task finds containers that have been finished for at least as long as
+# the duration specified in the `clean_container_log_rows_after` config setting,
+# and deletes their stdout, stderr, arv-mount, crunch-run, and crunchstat logs
+# from the logs table.
+
+namespace :db do
+ desc "Remove old container log entries from the logs table"
+ task delete_old_container_logs: :environment do
+ Log.select("logs.id").
+ joins("JOIN containers ON object_uuid = containers.uuid").
+ where("event_type in ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat') AND containers.log IS NOT NULL AND containers.finished_at < :age",
+ age: Rails.configuration.clean_container_log_rows_after.ago).
+ find_in_batches do |old_log_ids|
+ Log.where(id: old_log_ids.map(&:id)).delete_all
+ end
+ end
+end
diff --git a/services/api/test/fixtures/containers.yml b/services/api/test/fixtures/containers.yml
index 8e1a3ff..87098dc 100644
--- a/services/api/test/fixtures/containers.yml
+++ b/services/api/test/fixtures/containers.yml
@@ -136,7 +136,7 @@ requester_container:
auth_uuid: zzzzz-gj3su-077z32aux8dg2s1
failed_container:
- uuid: zzzzz-dz642-failedcntnr
+ uuid: zzzzz-dz642-failedcontainr1
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
state: Complete
exit_code: 33
@@ -151,3 +151,56 @@ failed_container:
runtime_constraints:
ram: 12000000000
vcpus: 4
+
+ancient_container_with_logs:
+ uuid: zzzzz-dz642-logscontainer01
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Complete
+ exit_code: 0
+ priority: 1
+ created_at: <%= 2.year.ago.to_s(:db) %>
+ updated_at: <%= 2.year.ago.to_s(:db) %>
+ container_image: test
+ cwd: test
+ output_path: test
+ command: ["echo", "hello"]
+ runtime_constraints:
+ ram: 12000000000
+ vcpus: 4
+ finished_at: <%= 2.year.ago.to_s(:db) %>
+ log: ea10d51bcf88862dbcc36eb292017dfd+45
+ output: test
+
+previous_container_with_logs:
+ uuid: zzzzz-dz642-logscontainer02
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Complete
+ exit_code: 0
+ priority: 1
+ created_at: <%= 1.month.ago.to_s(:db) %>
+ updated_at: <%= 1.month.ago.to_s(:db) %>
+ container_image: test
+ cwd: test
+ output_path: test
+ command: ["echo", "hello"]
+ runtime_constraints:
+ ram: 12000000000
+ vcpus: 4
+ finished_at: <%= 1.month.ago.to_s(:db) %>
+ log: ea10d51bcf88862dbcc36eb292017dfd+45
+ output: test
+
+running_container_with_logs:
+ uuid: zzzzz-dz642-logscontainer03
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Running
+ priority: 1
+ created_at: <%= 1.hour.ago.to_s(:db) %>
+ updated_at: <%= 1.hour.ago.to_s(:db) %>
+ container_image: test
+ cwd: test
+ output_path: test
+ command: ["echo", "hello"]
+ runtime_constraints:
+ ram: 12000000000
+ vcpus: 4
diff --git a/services/api/test/fixtures/logs.yml b/services/api/test/fixtures/logs.yml
index 2f45d69..d83cf96 100644
--- a/services/api/test/fixtures/logs.yml
+++ b/services/api/test/fixtures/logs.yml
@@ -141,3 +141,117 @@ crunchstat_for_ancient_job:
updated_at: 2013-11-07 23:33:42.347455000 Z
modified_at: 2013-11-07 23:33:42.347455000 Z
object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+stderr_for_ancient_container:
+ id: 12
+ uuid: zzzzz-57u5n-containerlog001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ object_uuid: zzzzz-dz642-logscontainer01
+ event_at: <%= 2.year.ago.to_s(:db) %>
+ event_type: stderr
+ summary: ~
+ properties:
+ text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+ cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+ 0.9900 sys'
+ created_at: <%= 2.year.ago.to_s(:db) %>
+ updated_at: <%= 2.year.ago.to_s(:db) %>
+ modified_at: <%= 2.year.ago.to_s(:db) %>
+ object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+crunchstat_for_ancient_container:
+ id: 13
+ uuid: zzzzz-57u5n-containerlog002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ object_uuid: zzzzz-dz642-logscontainer01
+ event_at: <%= 2.year.ago.to_s(:db) %>
+ event_type: crunchstat
+ summary: ~
+ properties:
+ text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+ cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+ 0.9900 sys'
+ created_at: <%= 2.year.ago.to_s(:db) %>
+ updated_at: <%= 2.year.ago.to_s(:db) %>
+ modified_at: <%= 2.year.ago.to_s(:db) %>
+ object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+stderr_for_previous_container:
+ id: 14
+ uuid: zzzzz-57u5n-containerlog003
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ object_uuid: zzzzz-dz642-logscontainer02
+ event_at: <%= 1.month.ago.to_s(:db) %>
+ event_type: stderr
+ summary: ~
+ properties:
+ text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+ cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+ 0.9900 sys'
+ created_at: <%= 1.month.ago.to_s(:db) %>
+ updated_at: <%= 1.month.ago.to_s(:db) %>
+ modified_at: <%= 1.month.ago.to_s(:db) %>
+ object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+crunchstat_for_previous_container:
+ id: 15
+ uuid: zzzzz-57u5n-containerlog004
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ object_uuid: zzzzz-dz642-logscontainer02
+ event_at: <%= 1.month.ago.to_s(:db) %>
+ event_type: crunchstat
+ summary: ~
+ properties:
+ text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+ cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+ 0.9900 sys'
+ created_at: <%= 1.month.ago.to_s(:db) %>
+ updated_at: <%= 1.month.ago.to_s(:db) %>
+ modified_at: <%= 1.month.ago.to_s(:db) %>
+ object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+stderr_for_running_container:
+ id: 16
+ uuid: zzzzz-57u5n-containerlog005
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ object_uuid: zzzzz-dz642-logscontainer03
+ event_at: <%= 1.hour.ago.to_s(:db) %>
+ event_type: crunchstat
+ summary: ~
+ properties:
+ text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+ cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+ 0.9900 sys'
+ created_at: <%= 1.hour.ago.to_s(:db) %>
+ updated_at: <%= 1.hour.ago.to_s(:db) %>
+ modified_at: <%= 1.hour.ago.to_s(:db) %>
+ object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
+
+crunchstat_for_running_container:
+ id: 17
+ uuid: zzzzz-57u5n-containerlog006
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_by_client_uuid: zzzzz-ozdt8-obw7foaks3qjyej
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ object_uuid: zzzzz-dz642-logscontainer03
+ event_at: <%= 1.hour.ago.to_s(:db) %>
+ event_type: crunchstat
+ summary: ~
+ properties:
+ text: '2013-11-07_23:33:41 zzzzz-8i9sb-ahd7cie8jah9qui 29610 1 stderr crunchstat:
+ cpu 1935.4300 user 59.4100 sys 8 cpus -- interval 10.0002 seconds 12.9900 user
+ 0.9900 sys'
+ created_at: <%= 1.hour.ago.to_s(:db) %>
+ updated_at: <%= 1.hour.ago.to_s(:db) %>
+ modified_at: <%= 1.hour.ago.to_s(:db) %>
+ object_owner_uuid: zzzzz-j7d0g-xurymjxw79nv3jz
diff --git a/services/api/test/tasks/delete_old_container_logs_test.rb b/services/api/test/tasks/delete_old_container_logs_test.rb
new file mode 100644
index 0000000..82a5752
--- /dev/null
+++ b/services/api/test/tasks/delete_old_container_logs_test.rb
@@ -0,0 +1,50 @@
+require 'test_helper'
+require 'rake'
+
+Rake.application.rake_require "tasks/delete_old_container_logs"
+Rake::Task.define_task(:environment)
+
+class DeleteOldContainerLogsTaskTest < ActiveSupport::TestCase
+ TASK_NAME = "db:delete_old_container_logs"
+
+ def log_uuids(*fixture_names)
+ fixture_names.map { |name| logs(name).uuid }
+ end
+
+ def run_with_expiry(clean_after)
+ Rails.configuration.clean_container_log_rows_after = clean_after
+ Rake::Task[TASK_NAME].reenable
+ Rake.application.invoke_task TASK_NAME
+ end
+
+ def check_log_existence(test_method, fixture_uuids)
+ uuids_now = Log.where("object_uuid LIKE :pattern AND event_type in ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat')", pattern: "%-dz642-%").map(&:uuid)
+ fixture_uuids.each do |expect_uuid|
+ send(test_method, uuids_now, expect_uuid)
+ end
+ end
+
+ test "delete all finished logs" do
+ uuids_to_keep = log_uuids(:stderr_for_running_container,
+ :crunchstat_for_running_container)
+ uuids_to_clean = log_uuids(:stderr_for_previous_container,
+ :crunchstat_for_previous_container,
+ :stderr_for_ancient_container,
+ :crunchstat_for_ancient_container)
+ run_with_expiry(1)
+ check_log_existence(:assert_includes, uuids_to_keep)
+ check_log_existence(:refute_includes, uuids_to_clean)
+ end
+
+ test "delete old finished logs" do
+ uuids_to_keep = log_uuids(:stderr_for_running_container,
+ :crunchstat_for_running_container,
+ :stderr_for_previous_container,
+ :crunchstat_for_previous_container)
+ uuids_to_clean = log_uuids(:stderr_for_ancient_container,
+ :crunchstat_for_ancient_container)
+ run_with_expiry(360.days)
+ check_log_existence(:assert_includes, uuids_to_keep)
+ check_log_existence(:refute_includes, uuids_to_clean)
+ end
+end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list