Merge pull request #337 from bdunne/nightly_rpm_cleaner

Add a purger for the nightly builds and run it before updating the RPM repo

Gemfile

@@ -1,5 +1,6 @@
 source 'https://rubygems.org'
 
+gem 'activesupport',              :require => false
 gem 'awesome_spawn',  ">=1.3.0"
 gem 'aws-sdk-s3'
 gem 'config'

bin/build.rb

@@ -52,6 +52,7 @@
 
 if opts[:update_rpm_repo]
   ManageIQ::RPMBuild::BuildUploader.new(:release => build_type == "release").upload
+  ManageIQ::RPMBuild::NightlyBuildPurger.new.run
   ManageIQ::RPMBuild::RpmRepo.new.update
 end
 

bin/nightly_build_purger.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.expand_path("../lib", __dir__)
+
+require 'manageiq-rpm_build'
+
+ManageIQ::RPMBuild::NightlyBuildPurger.new.run

lib/manageiq/rpm_build.rb

@@ -15,6 +15,7 @@ class String
 require 'manageiq/rpm_build/generate_ansible_venv'
 require 'manageiq/rpm_build/generate_gemset'
 require 'manageiq/rpm_build/generate_tar_files'
+require 'manageiq/rpm_build/nightly_build_purger'
 require 'manageiq/rpm_build/rpm_repo'
 require 'manageiq/rpm_build/setup_source_repos'
 

lib/manageiq/rpm_build/nightly_build_purger.rb

@@ -0,0 +1,55 @@
+module ManageIQ
+  module RPMBuild
+    class NightlyBuildPurger
+      include S3Common
+
+      def run
+        candidates = {}
+        keepers    = {}
+
+        client.list_objects(:bucket => OPTIONS.rpm_repository.s3_api.bucket, :prefix => File.join("builds", "manageiq-nightly")).flat_map(&:contents).each do |object|
+          package, timestamp    = package_timestamp_from_key(object.key)
+          candidates[package] ||= []
+          keepers[package]    ||= []
+
+          if recent?(timestamp)
+            keepers[package] << object.key
+            next
+          end
+
+          position = timestamp[2, 8].to_i # YYYYMMDDHHMMSS -> YYMMDDHH
+          candidates[package][position] = object.key
+        end
+
+        candidates.each_key do |package|
+          # Keep the most recent 7 nightly builds of any package
+          (candidates[package].compact[0..-7] - keepers[package]).each { |i| delete(i) }
+        end
+      end
+
+      private
+
+      def delete(key)
+        print "Deleting #{key}... "
+        client.delete_object(:bucket => OPTIONS.rpm_repository.s3_api.bucket, :key => key)
+        puts "done."
+      end
+
+      def now
+        @now ||= Time.now.utc
+      end
+
+      def package_timestamp_from_key(key)
+        key.match(/.*\/(.*)-.*(\d{14}).*/).captures
+      end
+
+      def recent?(timestamp)
+        require 'active_support'
+        require 'active_support/time'
+        require 'date'
+        # Keep anything 1.week or newer
+        DateTime.parse(timestamp) > 1.week.before(now)
+      end
+    end
+  end
+end

spec/lib/manageiq/rpm_build/nightly_build_purger_spec.rb

@@ -0,0 +1,19 @@
+RSpec.describe(ManageIQ::RPMBuild::NightlyBuildPurger) do
+  it("#package_timestamp_from_key (private)") do
+    expect(described_class.new.send(:package_timestamp_from_key, "builds/manageiq-nightly/manageiq-pods-13.0.0-20211004000013.el8.x86_64.rpm")).to eq(["manageiq-pods-13.0.0", "20211004000013"])
+    expect(described_class.new.send(:package_timestamp_from_key, "builds/manageiq-nightly/manageiq-ui-13.0.0-20211004000013.el8.x86_64.rpm")).to eq(["manageiq-ui-13.0.0", "20211004000013"])
+    expect(described_class.new.send(:package_timestamp_from_key, "builds/manageiq-nightly/manageiq-ui-13.0.0-0.1.20210705000025.el8.x86_64.rpm")).to eq(["manageiq-ui-13.0.0", "20210705000025"])
+  end
+
+  it("#recent? (private)") do
+    allow(subject).to receive(:now).and_return(Time.at(1668100000).utc) # 2022-11-10 17:06:40
+
+    expect(subject.send(:recent?, "20221110170640")).to eq(true) # Now
+    expect(subject.send(:recent?, "20221230170640")).to eq(true) # Future
+    expect(subject.send(:recent?, "20221104170640")).to eq(true) # 6 days ago
+    expect(subject.send(:recent?, "20221103170640")).to eq(false) # 1 week ago
+    expect(subject.send(:recent?, "20221027170640")).to eq(false) # 2 weeks ago
+    expect(subject.send(:recent?, "19700101000000")).to eq(false)
+    expect { subject.send(:recent?, "xyz") }.to raise_error(Date::Error)
+  end
+end