From 88dd01cf317130bc76644631e028495cf7863811 Mon Sep 17 00:00:00 2001 From: Ari Zellner Date: Tue, 14 Mar 2017 14:25:12 +0200 Subject: [PATCH] Add purge timer for archived entities --- app/models/container.rb | 1 + app/models/container/purging.rb | 20 ++++++++ app/models/container_definition.rb | 1 + app/models/container_definition/purging.rb | 20 ++++++++ app/models/container_group.rb | 1 + app/models/container_group/purging.rb | 20 ++++++++ app/models/container_image.rb | 6 +-- app/models/container_image/purging.rb | 20 ++++++++ app/models/container_project.rb | 1 + app/models/container_project/purging.rb | 20 ++++++++ app/models/miq_schedule_worker/jobs.rb | 8 ++++ app/models/miq_schedule_worker/runner.rb | 5 ++ config/settings.yml | 5 ++ spec/factories/container_definition.rb | 4 ++ spec/models/container/purging_spec.rb | 47 +++++++++++++++++++ .../container_definition/purging_spec.rb | 47 +++++++++++++++++++ spec/models/container_group/purging_spec.rb | 47 +++++++++++++++++++ spec/models/container_image/purging_spec.rb | 47 +++++++++++++++++++ spec/models/container_project/purging_spec.rb | 47 +++++++++++++++++++ 19 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 app/models/container/purging.rb create mode 100644 app/models/container_definition/purging.rb create mode 100644 app/models/container_group/purging.rb create mode 100644 app/models/container_image/purging.rb create mode 100644 app/models/container_project/purging.rb create mode 100644 spec/factories/container_definition.rb create mode 100644 spec/models/container/purging_spec.rb create mode 100644 spec/models/container_definition/purging_spec.rb create mode 100644 spec/models/container_group/purging_spec.rb create mode 100644 spec/models/container_image/purging_spec.rb create mode 100644 spec/models/container_project/purging_spec.rb diff --git a/app/models/container.rb b/app/models/container.rb index 2cbe638192b..f2df373aa62 100644 --- a/app/models/container.rb +++ b/app/models/container.rb @@ -2,6 +2,7 @@ class Container < ApplicationRecord include SupportsFeatureMixin include NewWithTypeStiMixin include ArchivedMixin + include_concern 'Purging' has_one :container_group, :through => :container_definition belongs_to :ext_management_system, :foreign_key => :ems_id diff --git a/app/models/container/purging.rb b/app/models/container/purging.rb new file mode 100644 index 00000000000..e3b7a0f235b --- /dev/null +++ b/app/models/container/purging.rb @@ -0,0 +1,20 @@ +class Container < ApplicationRecord + module Purging + extend ActiveSupport::Concern + include PurgingMixin + + module ClassMethods + def purge_date + ::Settings.container_entities.history.keep_archived_entities.to_i_with_method.seconds.ago.utc + end + + def purge_window_size + ::Settings.container_entities.history.purge_window_size + end + + def purge_scope(older_than) + where(arel_table[:deleted_on].lteq(older_than)) + end + end + end +end diff --git a/app/models/container_definition.rb b/app/models/container_definition.rb index 0fb6e95b1a4..d5d43b75fa7 100644 --- a/app/models/container_definition.rb +++ b/app/models/container_definition.rb @@ -1,5 +1,6 @@ class ContainerDefinition < ApplicationRecord include ArchivedMixin + include_concern 'Purging' # :name, :image, :image_pull_policy, :memory, :cpu belongs_to :container_group belongs_to :ext_management_system, :foreign_key => :ems_id diff --git a/app/models/container_definition/purging.rb b/app/models/container_definition/purging.rb new file mode 100644 index 00000000000..e4f6afaade3 --- /dev/null +++ b/app/models/container_definition/purging.rb @@ -0,0 +1,20 @@ +class ContainerDefinition < ApplicationRecord + module Purging + extend ActiveSupport::Concern + include PurgingMixin + + module ClassMethods + def purge_date + ::Settings.container_entities.history.keep_archived_entities.to_i_with_method.seconds.ago.utc + end + + def purge_window_size + ::Settings.container_entities.history.purge_window_size + end + + def purge_scope(older_than) + where(arel_table[:deleted_on].lteq(older_than)) + end + end + end +end diff --git a/app/models/container_group.rb b/app/models/container_group.rb index 01bd1de2cba..089f46b436e 100644 --- a/app/models/container_group.rb +++ b/app/models/container_group.rb @@ -6,6 +6,7 @@ class ContainerGroup < ApplicationRecord include NewWithTypeStiMixin include TenantIdentityMixin include ArchivedMixin + include_concern 'Purging' # :name, :uid, :creation_timestamp, :resource_version, :namespace # :labels, :restart_policy, :dns_policy diff --git a/app/models/container_group/purging.rb b/app/models/container_group/purging.rb new file mode 100644 index 00000000000..9056a65c763 --- /dev/null +++ b/app/models/container_group/purging.rb @@ -0,0 +1,20 @@ +class ContainerGroup < ApplicationRecord + module Purging + extend ActiveSupport::Concern + include PurgingMixin + + module ClassMethods + def purge_date + ::Settings.container_entities.history.keep_archived_entities.to_i_with_method.seconds.ago.utc + end + + def purge_window_size + ::Settings.container_entities.history.purge_window_size + end + + def purge_scope(older_than) + where(arel_table[:deleted_on].lteq(older_than)) + end + end + end +end diff --git a/app/models/container_image.rb b/app/models/container_image.rb index b24710cac4b..426fc52438e 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -4,7 +4,8 @@ class ContainerImage < ApplicationRecord include ScanningMixin include TenantIdentityMixin include CustomAttributeMixin - + include ArchivedMixin + include_concern 'Purging' DOCKER_IMAGE_PREFIX = "docker://" DOCKER_PULLABLE_PREFIX = "docker-pullable://".freeze @@ -27,9 +28,6 @@ class ContainerImage < ApplicationRecord serialize :exposed_ports, Hash serialize :environment_variables, Hash - # Needed for scanning & tagging action - delegate :my_zone, :to => :ext_management_system - acts_as_miq_taggable virtual_column :display_registry, :type => :string diff --git a/app/models/container_image/purging.rb b/app/models/container_image/purging.rb new file mode 100644 index 00000000000..2858fc28e62 --- /dev/null +++ b/app/models/container_image/purging.rb @@ -0,0 +1,20 @@ +class ContainerImage < ApplicationRecord + module Purging + extend ActiveSupport::Concern + include PurgingMixin + + module ClassMethods + def purge_date + ::Settings.container_entities.history.keep_archived_entities.to_i_with_method.seconds.ago.utc + end + + def purge_window_size + ::Settings.container_entities.history.purge_window_size + end + + def purge_scope(older_than) + where(arel_table[:deleted_on].lteq(older_than)) + end + end + end +end diff --git a/app/models/container_project.rb b/app/models/container_project.rb index 02bf6ba6c80..a6a3b9e96d1 100644 --- a/app/models/container_project.rb +++ b/app/models/container_project.rb @@ -2,6 +2,7 @@ class ContainerProject < ApplicationRecord include SupportsFeatureMixin include CustomAttributeMixin include ArchivedMixin + include_concern 'Purging' belongs_to :ext_management_system, :foreign_key => "ems_id" has_many :container_groups has_many :container_routes diff --git a/app/models/container_project/purging.rb b/app/models/container_project/purging.rb new file mode 100644 index 00000000000..7660f1046d1 --- /dev/null +++ b/app/models/container_project/purging.rb @@ -0,0 +1,20 @@ +class ContainerProject < ApplicationRecord + module Purging + extend ActiveSupport::Concern + include PurgingMixin + + module ClassMethods + def purge_date + ::Settings.container_entities.history.keep_archived_entities.to_i_with_method.seconds.ago.utc + end + + def purge_window_size + ::Settings.container_entities.history.purge_window_size + end + + def purge_scope(older_than) + where(arel_table[:deleted_on].lteq(older_than)) + end + end + end +end diff --git a/app/models/miq_schedule_worker/jobs.rb b/app/models/miq_schedule_worker/jobs.rb index 0eda81cb2a3..b0800e66149 100644 --- a/app/models/miq_schedule_worker/jobs.rb +++ b/app/models/miq_schedule_worker/jobs.rb @@ -121,6 +121,14 @@ def miq_report_result_purge_timer queue_work(:class_name => "MiqReportResult", :method_name => "purge_timer", :zone => nil) end + def archived_entities_purge_timer + queue_work(:class_name => "Container", :method_name => "purge_timer", :zone => nil) + queue_work(:class_name => "ContainerGroup", :method_name => "purge_timer", :zone => nil) + queue_work(:class_name => "ContainerImage", :method_name => "purge_timer", :zone => nil) + queue_work(:class_name => "ContainerProject", :method_name => "purge_timer", :zone => nil) + queue_work(:class_name => "ContainerDefinition", :method_name => "purge_timer", :zone => nil) + end + def storage_refresh_metrics queue_work( :class_name => "StorageManager", diff --git a/app/models/miq_schedule_worker/runner.rb b/app/models/miq_schedule_worker/runner.rb index bd119743038..2aca8c407e6 100644 --- a/app/models/miq_schedule_worker/runner.rb +++ b/app/models/miq_schedule_worker/runner.rb @@ -221,6 +221,11 @@ def schedules_for_scheduler_role enqueue :miq_report_result_purge_timer end + every = worker_settings[:container_entities_purge_interval] + scheduler.schedule_every(every, :first_in => every) do + enqueue :archived_entities_purge_timer + end + # Schedule every 24 hours at = worker_settings[:storage_file_collection_time_utc] if Time.now.strftime("%Y-%m-%d #{at}").to_time(:utc) < Time.now.utc diff --git a/config/settings.yml b/config/settings.yml index 47020a3187a..c627a48eec7 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -1031,6 +1031,10 @@ :history: :keep_policy_events: 6.months :purge_window_size: 1000 +:container_entities: + :history: + :keep_archived_entities: 6.months + :purge_window_size: 1000 :product: :maindb: ExtManagementSystem :container_deployment_wizard: false @@ -1363,6 +1367,7 @@ :poll_method: :normal :queue_timeout: 120.minutes :schedule_worker: + :container_entities_purge_interval: 1.day :authentication_check_interval: 1.hour :chargeback_generation_interval: 1.day :chargeback_generation_time_utc: 01:00:00 diff --git a/spec/factories/container_definition.rb b/spec/factories/container_definition.rb new file mode 100644 index 00000000000..8561e0eedac --- /dev/null +++ b/spec/factories/container_definition.rb @@ -0,0 +1,4 @@ +FactoryGirl.define do + factory :container_definition do + end +end diff --git a/spec/models/container/purging_spec.rb b/spec/models/container/purging_spec.rb new file mode 100644 index 00000000000..9a1e7991ce5 --- /dev/null +++ b/spec/models/container/purging_spec.rb @@ -0,0 +1,47 @@ +describe Container do + context "::Purging" do + context ".purge_queue" do + before do + EvmSpecHelper.create_guid_miq_server_zone + end + let(:purge_time) { (Time.zone.now + 10).round } + + it "submits to the queue" do + expect(described_class).to receive(:purge_date).and_return(purge_time) + described_class.purge_timer + + q = MiqQueue.all + expect(q.length).to eq(1) + expect(q.first).to have_attributes( + :class_name => described_class.name, + :method_name => "purge_by_date", + :args => [purge_time] + ) + end + end + + context ".purge" do + let(:deleted_date) { 6.months.ago } + + before do + @old_container = FactoryGirl.create(:container, :deleted_on => deleted_date - 1.day) + @purge_date_container = FactoryGirl.create(:container, :deleted_on => deleted_date) + @new_container = FactoryGirl.create(:container, :deleted_on => deleted_date + 1.day) + end + + def assert_unpurged_ids(unpurged_ids) + expect(described_class.order(:id).pluck(:id)).to eq(Array(unpurged_ids).sort) + end + + it "purge_date and older" do + described_class.purge(deleted_date) + assert_unpurged_ids(@new_container.id) + end + + it "with a window" do + described_class.purge(deleted_date, 1) + assert_unpurged_ids(@new_container.id) + end + end + end +end diff --git a/spec/models/container_definition/purging_spec.rb b/spec/models/container_definition/purging_spec.rb new file mode 100644 index 00000000000..954f76f7684 --- /dev/null +++ b/spec/models/container_definition/purging_spec.rb @@ -0,0 +1,47 @@ +describe ContainerDefinition do + context "::Purging" do + context ".purge_queue" do + before do + EvmSpecHelper.create_guid_miq_server_zone + end + let(:purge_time) { (Time.zone.now + 10).round } + + it "submits to the queue" do + expect(described_class).to receive(:purge_date).and_return(purge_time) + described_class.purge_timer + + q = MiqQueue.all + expect(q.length).to eq(1) + expect(q.first).to have_attributes( + :class_name => described_class.name, + :method_name => "purge_by_date", + :args => [purge_time] + ) + end + end + + context ".purge" do + let(:deleted_date) { 6.months.ago } + + before do + @old_container_def = FactoryGirl.create(:container_definition, :deleted_on => deleted_date - 1.day) + @purge_date_container_def = FactoryGirl.create(:container_definition, :deleted_on => deleted_date) + @new_container_def = FactoryGirl.create(:container_definition, :deleted_on => deleted_date + 1.day) + end + + def assert_unpurged_ids(unpurged_ids) + expect(described_class.order(:id).pluck(:id)).to eq(Array(unpurged_ids).sort) + end + + it "purge_date and older" do + described_class.purge(deleted_date) + assert_unpurged_ids(@new_container_def.id) + end + + it "with a window" do + described_class.purge(deleted_date, 1) + assert_unpurged_ids(@new_container_def.id) + end + end + end +end diff --git a/spec/models/container_group/purging_spec.rb b/spec/models/container_group/purging_spec.rb new file mode 100644 index 00000000000..c0711c46894 --- /dev/null +++ b/spec/models/container_group/purging_spec.rb @@ -0,0 +1,47 @@ +describe ContainerGroup do + context "::Purging" do + context ".purge_queue" do + before do + EvmSpecHelper.create_guid_miq_server_zone + end + let(:purge_time) { (Time.zone.now + 10).round } + + it "submits to the queue" do + expect(described_class).to receive(:purge_date).and_return(purge_time) + described_class.purge_timer + + q = MiqQueue.all + expect(q.length).to eq(1) + expect(q.first).to have_attributes( + :class_name => described_class.name, + :method_name => "purge_by_date", + :args => [purge_time] + ) + end + end + + context ".purge" do + let(:deleted_date) { 6.months.ago } + + before do + @old_container_group = FactoryGirl.create(:container_group, :deleted_on => deleted_date - 1.day) + @purge_date_container_group = FactoryGirl.create(:container_group, :deleted_on => deleted_date) + @new_container_group = FactoryGirl.create(:container_group, :deleted_on => deleted_date + 1.day) + end + + def assert_unpurged_ids(unpurged_ids) + expect(described_class.order(:id).pluck(:id)).to eq(Array(unpurged_ids).sort) + end + + it "purge_date and older" do + described_class.purge(deleted_date) + assert_unpurged_ids(@new_container_group.id) + end + + it "with a window" do + described_class.purge(deleted_date, 1) + assert_unpurged_ids(@new_container_group.id) + end + end + end +end diff --git a/spec/models/container_image/purging_spec.rb b/spec/models/container_image/purging_spec.rb new file mode 100644 index 00000000000..9999328ecf2 --- /dev/null +++ b/spec/models/container_image/purging_spec.rb @@ -0,0 +1,47 @@ +describe ContainerImage do + context "::Purging" do + context ".purge_queue" do + before do + EvmSpecHelper.create_guid_miq_server_zone + end + let(:purge_time) { (Time.zone.now + 10).round } + + it "submits to the queue" do + expect(described_class).to receive(:purge_date).and_return(purge_time) + described_class.purge_timer + + q = MiqQueue.all + expect(q.length).to eq(1) + expect(q.first).to have_attributes( + :class_name => described_class.name, + :method_name => "purge_by_date", + :args => [purge_time] + ) + end + end + + context ".purge" do + let(:deleted_date) { 6.months.ago } + + before do + @old_container_image = FactoryGirl.create(:container_image, :deleted_on => deleted_date - 1.day) + @purge_date_container_image = FactoryGirl.create(:container_image, :deleted_on => deleted_date) + @new_container_image = FactoryGirl.create(:container_image, :deleted_on => deleted_date + 1.day) + end + + def assert_unpurged_ids(unpurged_ids) + expect(described_class.order(:id).pluck(:id)).to eq(Array(unpurged_ids).sort) + end + + it "purge_date and older" do + described_class.purge(deleted_date) + assert_unpurged_ids(@new_container_image.id) + end + + it "with a window" do + described_class.purge(deleted_date, 1) + assert_unpurged_ids(@new_container_image.id) + end + end + end +end diff --git a/spec/models/container_project/purging_spec.rb b/spec/models/container_project/purging_spec.rb new file mode 100644 index 00000000000..ef0129dbc8c --- /dev/null +++ b/spec/models/container_project/purging_spec.rb @@ -0,0 +1,47 @@ +describe ContainerProject do + context "::Purging" do + context ".purge_queue" do + before do + EvmSpecHelper.create_guid_miq_server_zone + end + let(:purge_time) { (Time.zone.now + 10).round } + + it "submits to the queue" do + expect(described_class).to receive(:purge_date).and_return(purge_time) + described_class.purge_timer + + q = MiqQueue.all + expect(q.length).to eq(1) + expect(q.first).to have_attributes( + :class_name => described_class.name, + :method_name => "purge_by_date", + :args => [purge_time] + ) + end + end + + context ".purge" do + let(:deleted_date) { 6.months.ago } + + before do + @old_container_project = FactoryGirl.create(:container_project, :deleted_on => deleted_date - 1.day) + @purge_date_container_project = FactoryGirl.create(:container_project, :deleted_on => deleted_date) + @new_container_project = FactoryGirl.create(:container_project, :deleted_on => deleted_date + 1.day) + end + + def assert_unpurged_ids(unpurged_ids) + expect(described_class.order(:id).pluck(:id)).to eq(Array(unpurged_ids).sort) + end + + it "purge_date and older" do + described_class.purge(deleted_date) + assert_unpurged_ids(@new_container_project.id) + end + + it "with a window" do + described_class.purge(deleted_date, 1) + assert_unpurged_ids(@new_container_project.id) + end + end + end +end