From d7dbc9dfc10d187cace0362da02eae4d426be340 Mon Sep 17 00:00:00 2001 From: Vinaykumar Bhat Date: Sun, 21 Jun 2020 21:58:06 -0700 Subject: [PATCH] auto deletion of apply logs based on sysvars Summary: Apply logs are the new trx log on raft followers. We should be aggressive in reclaiming space consumed by apply logs. This diff adds the support for auto deletion of apply logs. Approach: 1. Two new sysvars are added apply_log_retention_num (defaulted to 10 files) and apply_log_retention_duration (defaulted to 15 mins). These impose (soft) limits on minimum number of apply logs that need to be retained (in terms of 'number-of-files') and minimum duration of retention (in terms of minutes since last modified). 2. Auto deletion is triggered whenever an apply log gets rotated after commit of a trx (in ordered_commit) as part of max_binlog_size check. Auto deletion will trigger if there are more than "apply_log_retention_num" number of files. The files which were last modified more than "apply_log_retention_duration" minutes in the past will be auto deleted (provided these files are not being actively used currently) 3. To keep things simple, the new limits are only soft limits. Pathological cases like a user triggering multiple 'flush logs' and system not seeing any trx activity for a prolonged period is not handled. Reviewed By: anirbanr-fb Differential Revision: D22095395 fbshipit-source-id: 29a4040cdcb --- .../r/mysqld--help-notwin-profiling.result | 7 + mysql-test/r/mysqld--help-notwin.result | 7 + .../r/rpl_raft_auto_purge_apply_logs.result | 343 ++++++++++++++++++ ..._raft_auto_purge_apply_logs_restart.result | 164 +++++++++ .../rpl_raft_auto_purge_apply_logs-master.opt | 2 + .../rpl_raft_auto_purge_apply_logs-slave.opt | 2 + .../t/rpl_raft_auto_purge_apply_logs.test | 176 +++++++++ ...t_auto_purge_apply_logs_restart-master.opt | 2 + ...ft_auto_purge_apply_logs_restart-slave.opt | 2 + ...pl_raft_auto_purge_apply_logs_restart.test | 89 +++++ .../apply_log_retention_duration_basic.result | 17 + .../r/apply_log_retention_num_basic.result | 17 + .../t/apply_log_retention_duration_basic.test | 28 ++ .../t/apply_log_retention_num_basic.test | 28 ++ sql/binlog.cc | 137 ++++++- sql/binlog.h | 20 +- sql/mysqld.cc | 2 + sql/mysqld.h | 2 + sql/sys_vars.cc | 11 + 19 files changed, 1048 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs.result create mode 100644 mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs_restart.result create mode 100644 mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs-master.opt create mode 100644 mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs-slave.opt create mode 100644 mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs.test create mode 100644 mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart-master.opt create mode 100644 mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart-slave.opt create mode 100644 mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart.test create mode 100644 mysql-test/suite/sys_vars/r/apply_log_retention_duration_basic.result create mode 100644 mysql-test/suite/sys_vars/r/apply_log_retention_num_basic.result create mode 100644 mysql-test/suite/sys_vars/t/apply_log_retention_duration_basic.test create mode 100644 mysql-test/suite/sys_vars/t/apply_log_retention_num_basic.test diff --git a/mysql-test/r/mysqld--help-notwin-profiling.result b/mysql-test/r/mysqld--help-notwin-profiling.result index c10fad046bd7..a16cbdcb7a65 100644 --- a/mysql-test/r/mysqld--help-notwin-profiling.result +++ b/mysql-test/r/mysqld--help-notwin-profiling.result @@ -61,6 +61,11 @@ The following options may be given as the first argument: --apply-log-index=name The location and name to use for the file that keeps a list of the last apply logs for raft + --apply-log-retention-duration[=#] + Minimum duration (mins) that apply logs need to be + retained. + --apply-log-retention-num[=#] + Minimum number of apply logs that need to be retained. --async-query-counter Parsing async query info and collection of async query statistics @@ -2296,6 +2301,8 @@ allow-noncurrent-db-rw ON allow-suspicious-udfs FALSE apply-log (No default value) apply-log-index (No default value) +apply-log-retention-duration 15 +apply-log-retention-num 10 async-query-counter FALSE audit-fb-json-functions audit-instrumented-event AUDIT_OFF diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index d0bdf9af8e38..9512d0de6d15 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -61,6 +61,11 @@ The following options may be given as the first argument: --apply-log-index=name The location and name to use for the file that keeps a list of the last apply logs for raft + --apply-log-retention-duration[=#] + Minimum duration (mins) that apply logs need to be + retained. + --apply-log-retention-num[=#] + Minimum number of apply logs that need to be retained. --async-query-counter Parsing async query info and collection of async query statistics @@ -2294,6 +2299,8 @@ allow-noncurrent-db-rw ON allow-suspicious-udfs FALSE apply-log (No default value) apply-log-index (No default value) +apply-log-retention-duration 15 +apply-log-retention-num 10 async-query-counter FALSE audit-fb-json-functions audit-instrumented-event AUDIT_OFF diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs.result b/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs.result new file mode 100644 index 000000000000..533cba4f1b8e --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs.result @@ -0,0 +1,343 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +create table t1 (a int primary key); +"Inserting rows into t1 on server_1" +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +select count(*) from t1; +count(*) +20 +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +"Verifying apply logs on follower: server_2" +show binary logs; +Log_name File_size +apply-logs-13001.000001 # +apply-logs-13001.000002 # +apply-logs-13001.000003 # +apply-logs-13001.000004 # +apply-logs-13001.000005 # +apply-logs-13001.000006 # +apply-logs-13001.000007 # +apply-logs-13001.000008 # +apply-logs-13001.000009 # +apply-logs-13001.000010 # +apply-logs-13001.000011 # +apply-logs-13001.000012 # +apply-logs-13001.000013 # +apply-logs-13001.000014 # +apply-logs-13001.000015 # +apply-logs-13001.000016 # +apply-logs-13001.000017 # +apply-logs-13001.000018 # +apply-logs-13001.000019 # +apply-logs-13001.000020 # +apply-logs-13001.000021 # +apply-logs-13001.000022 # +apply-logs-13001.000023 # +"Verifying apply logs on follower: server_3" +show binary logs; +Log_name File_size +apply-logs-13002.000001 # +apply-logs-13002.000002 # +apply-logs-13002.000003 # +apply-logs-13002.000004 # +apply-logs-13002.000005 # +apply-logs-13002.000006 # +apply-logs-13002.000007 # +apply-logs-13002.000008 # +apply-logs-13002.000009 # +apply-logs-13002.000010 # +apply-logs-13002.000011 # +apply-logs-13002.000012 # +apply-logs-13002.000013 # +apply-logs-13002.000014 # +apply-logs-13002.000015 # +apply-logs-13002.000016 # +apply-logs-13002.000017 # +apply-logs-13002.000018 # +apply-logs-13002.000019 # +apply-logs-13002.000020 # +apply-logs-13002.000021 # +apply-logs-13002.000022 # +apply-logs-13002.000023 # +"Sleep for 70s to expire the logs" +select sleep(70); +sleep(70) +0 +"Inserting another row on leader: server_1. This will trigger auto purge on followers" +insert into t1 values(21); +select count(*) from t1; +count(*) +21 +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +"Verifying apply logs on follower: server_2" +select count(*) from t1; +count(*) +21 +show binary logs; +Log_name File_size +apply-logs-13001.000015 # +apply-logs-13001.000016 # +apply-logs-13001.000017 # +apply-logs-13001.000018 # +apply-logs-13001.000019 # +apply-logs-13001.000020 # +apply-logs-13001.000021 # +apply-logs-13001.000022 # +apply-logs-13001.000023 # +apply-logs-13001.000024 # +"Verifying apply logs on follower: server_3" +select count(*) from t1; +count(*) +21 +show binary logs; +Log_name File_size +apply-logs-13002.000015 # +apply-logs-13002.000016 # +apply-logs-13002.000017 # +apply-logs-13002.000018 # +apply-logs-13002.000019 # +apply-logs-13002.000020 # +apply-logs-13002.000021 # +apply-logs-13002.000022 # +apply-logs-13002.000023 # +apply-logs-13002.000024 # +"Transfering leadership: server_1 -> server_2" +include/raft_promote_to_leader.inc +select sleep(10); +sleep(10) +0 +select variable_value from information_schema.global_status where variable_name = 'Rpl_raft_role'; +variable_value +LEADER +"Inserting some rows into t1 from leader: server_2" +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +"Sleep for 70s to expire the logs" +select sleep(70); +sleep(70) +0 +"Inserting another row on leader: server_2. This will trigger auto purge on followers" +insert into t1 values(41); +select count(*) from t1; +count(*) +41 +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +"Verifying apply logs on follower: server_1" +select count(*) from t1; +count(*) +41 +show binary logs; +Log_name File_size +apply-logs-13000.000012 # +apply-logs-13000.000013 # +apply-logs-13000.000014 # +apply-logs-13000.000015 # +apply-logs-13000.000016 # +apply-logs-13000.000017 # +apply-logs-13000.000018 # +apply-logs-13000.000019 # +apply-logs-13000.000020 # +apply-logs-13000.000021 # +"Verifying apply logs on follower: server_3" +select count(*) from t1; +count(*) +41 +show binary logs; +Log_name File_size +apply-logs-13002.000036 # +apply-logs-13002.000037 # +apply-logs-13002.000038 # +apply-logs-13002.000039 # +apply-logs-13002.000040 # +apply-logs-13002.000041 # +apply-logs-13002.000042 # +apply-logs-13002.000043 # +apply-logs-13002.000044 # +apply-logs-13002.000045 # +"Transfering leadership: server_2 -> server_1" +include/raft_promote_to_leader.inc +select variable_value from information_schema.global_status where variable_name = 'Rpl_raft_role'; +variable_value +LEADER +"Inserting more rows into t1 from server_1" +select sleep(10); +sleep(10) +0 +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +"Sleep for 70s to expire the logs" +select sleep(70); +sleep(70) +0 +"Inserting another row on leader: server_1. This will trigger auto purge on followers" +insert into t1 values(61); +select count(*) from t1; +count(*) +61 +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +"Verifying apply logs on follower: server_2" +select count(*) from t1; +count(*) +61 +show binary logs; +Log_name File_size +apply-logs-13001.000012 # +apply-logs-13001.000013 # +apply-logs-13001.000014 # +apply-logs-13001.000015 # +apply-logs-13001.000016 # +apply-logs-13001.000017 # +apply-logs-13001.000018 # +apply-logs-13001.000019 # +apply-logs-13001.000020 # +apply-logs-13001.000021 # +"Verifying apply logs on follower: server_3" +select count(*) from t1; +count(*) +61 +show binary logs; +Log_name File_size +apply-logs-13002.000057 # +apply-logs-13002.000058 # +apply-logs-13002.000059 # +apply-logs-13002.000060 # +apply-logs-13002.000061 # +apply-logs-13002.000062 # +apply-logs-13002.000063 # +apply-logs-13002.000064 # +apply-logs-13002.000065 # +apply-logs-13002.000066 # +drop table t1; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs_restart.result b/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs_restart.result new file mode 100644 index 000000000000..1ed4f30fc240 --- /dev/null +++ b/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs_restart.result @@ -0,0 +1,164 @@ +include/raft_3_node.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +call mtr.add_suppression("Engine has seen trxs till file .*"); +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role LEADER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +show status like 'rpl_raft_role'; +Variable_name Value +Rpl_raft_role FOLLOWER +create table t1 (a int primary key) engine=innodb; +"Inserting rows into t1 on server_1" +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +select count(*) from t1; +count(*) +12 +"Verifying apply logs on follower: server_2" +select count(*) from t1; +count(*) +12 +show binary logs; +Log_name File_size +apply-logs-13001.000001 # +apply-logs-13001.000002 # +apply-logs-13001.000003 # +apply-logs-13001.000004 # +apply-logs-13001.000005 # +apply-logs-13001.000006 # +apply-logs-13001.000007 # +apply-logs-13001.000008 # +apply-logs-13001.000009 # +apply-logs-13001.000010 # +apply-logs-13001.000011 # +apply-logs-13001.000012 # +apply-logs-13001.000013 # +apply-logs-13001.000014 # +apply-logs-13001.000015 # +apply-logs-13001.000016 # +"Verifying apply logs on follower: server_3" +select count(*) from t1; +count(*) +12 +show binary logs; +Log_name File_size +apply-logs-13002.000001 # +apply-logs-13002.000002 # +apply-logs-13002.000003 # +apply-logs-13002.000004 # +apply-logs-13002.000005 # +apply-logs-13002.000006 # +apply-logs-13002.000007 # +apply-logs-13002.000008 # +apply-logs-13002.000009 # +apply-logs-13002.000010 # +apply-logs-13002.000011 # +apply-logs-13002.000012 # +apply-logs-13002.000013 # +apply-logs-13002.000014 # +apply-logs-13002.000015 # +apply-logs-13002.000016 # +"Restarting follower: server_2" +include/rpl_restart_server.inc [server_number=2] +include/start_slave_sql.inc +"Verifying apply logs on follower: server_2" +show binary logs; +Log_name File_size +apply-logs-13001.000001 # +apply-logs-13001.000002 # +apply-logs-13001.000003 # +apply-logs-13001.000004 # +apply-logs-13001.000005 # +apply-logs-13001.000006 # +apply-logs-13001.000007 # +apply-logs-13001.000008 # +apply-logs-13001.000009 # +apply-logs-13001.000010 # +apply-logs-13001.000011 # +apply-logs-13001.000012 # +apply-logs-13001.000013 # +apply-logs-13001.000014 # +apply-logs-13001.000015 # +apply-logs-13001.000016 # +apply-logs-13001.000017 # +select count(*) from t1; +count(*) +12 +"Sleep for 70s to expire apply logs on followers" +select sleep(70); +sleep(70) +0 +"Inserting one row in leader: server_1. This should trigger auto purge of apply logs on followers" +insert into t1 values(12); +select count(*) from t1; +count(*) +13 +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +"Verifying apply logs on follower: server_2" +select count(*) from t1; +count(*) +13 +show binary logs; +Log_name File_size +apply-logs-13001.000009 # +apply-logs-13001.000010 # +apply-logs-13001.000011 # +apply-logs-13001.000012 # +apply-logs-13001.000013 # +apply-logs-13001.000014 # +apply-logs-13001.000015 # +apply-logs-13001.000016 # +apply-logs-13001.000017 # +apply-logs-13001.000018 # +"Verifying apply logs on follower: server_3" +select count(*) from t1; +count(*) +13 +show binary logs; +Log_name File_size +apply-logs-13002.000008 # +apply-logs-13002.000009 # +apply-logs-13002.000010 # +apply-logs-13002.000011 # +apply-logs-13002.000012 # +apply-logs-13002.000013 # +apply-logs-13002.000014 # +apply-logs-13002.000015 # +apply-logs-13002.000016 # +apply-logs-13002.000017 # +drop table t1; +include/sync_slave_sql_with_master.inc +include/sync_slave_sql_with_master.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs-master.opt b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs-master.opt new file mode 100644 index 000000000000..5c4b611c64fe --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs-master.opt @@ -0,0 +1,2 @@ +--loose-debug=+d,force_rotate --apply_log_retention_num=10 +--apply_log_retention_duration=1 diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs-slave.opt b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs-slave.opt new file mode 100644 index 000000000000..5c4b611c64fe --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs-slave.opt @@ -0,0 +1,2 @@ +--loose-debug=+d,force_rotate --apply_log_retention_num=10 +--apply_log_retention_duration=1 diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs.test b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs.test new file mode 100644 index 000000000000..33fc3c2edaad --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs.test @@ -0,0 +1,176 @@ +source ../include/raft_3_node.inc; + +connection server_1; +show status like 'rpl_raft_role'; + +connection server_2; +show status like 'rpl_raft_role'; + +connection server_3; +show status like 'rpl_raft_role'; + +connection server_1; +create table t1 (a int primary key); + +# Insert some rows into t1. This will rotate the apply log on follower for every +# insert +echo "Inserting rows into t1 on server_1"; +--let $j = 1 +--disable_query_log +while ($j <= 20) +{ + connection server_1; + eval insert into t1 values($j); + let $sync_slave_connection= server_2; + source include/sync_slave_sql_with_master.inc; + let $sync_slave_connection= server_3; + source include/sync_slave_sql_with_master.inc; + + --inc $j +} +--enable_query_log +select count(*) from t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +echo "Verifying apply logs on follower: server_2"; +connection server_2; +let $binlog= query_get_value(SHOW MASTER STATUS, File, 1); +source include/show_binary_logs.inc; + +echo "Verifying apply logs on follower: server_3"; +connection server_3; +source include/show_binary_logs.inc; + +echo "Sleep for 70s to expire the logs"; +select sleep(70); + +connection server_1; +echo "Inserting another row on leader: server_1. This will trigger auto purge on followers"; +eval insert into t1 values($j); +--inc $j +select count(*) from t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +connection server_2; +echo "Verifying apply logs on follower: server_2"; +select count(*) from t1; +source include/show_binary_logs.inc; + +connection server_3; +echo "Verifying apply logs on follower: server_3"; +select count(*) from t1; +source include/show_binary_logs.inc; + +echo "Transfering leadership: server_1 -> server_2"; +let $rpl_raft_leader_number= 2; +source ../include/raft_promote_to_leader.inc; + +connection server_2; +# Sleep for mysql master to be setup +select sleep(10); +select variable_value from information_schema.global_status where variable_name = 'Rpl_raft_role'; + +echo "Inserting some rows into t1 from leader: server_2"; +--disable_query_log +while ($j <= 40) +{ + connection server_2; + eval insert into t1 values($j); + let $sync_slave_connection= server_1; + source include/sync_slave_sql_with_master.inc; + let $sync_slave_connection= server_3; + source include/sync_slave_sql_with_master.inc; + + --inc $j +} +--enable_query_log + +echo "Sleep for 70s to expire the logs"; +select sleep(70); + +echo "Inserting another row on leader: server_2. This will trigger auto purge on followers"; +connection server_2; +eval insert into t1 values($j); +select count(*) from t1; +--inc $j + +let $sync_slave_connection= server_1; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +connection server_1; +echo "Verifying apply logs on follower: server_1"; +select count(*) from t1; +source include/show_binary_logs.inc; + +connection server_3; +echo "Verifying apply logs on follower: server_3"; +select count(*) from t1; +source include/show_binary_logs.inc; + +connection server_1; +echo "Transfering leadership: server_2 -> server_1"; +let $rpl_raft_leader_number= 1; +source ../include/raft_promote_to_leader.inc; +select variable_value from information_schema.global_status where variable_name = 'Rpl_raft_role'; + +echo "Inserting more rows into t1 from server_1"; +connection server_1; +# Sleep for mysql master to be setup +select sleep(10); +--disable_query_log +while ($j <= 60) +{ + connection server_1; + eval insert into t1 values($j); + let $sync_slave_connection= server_2; + source include/sync_slave_sql_with_master.inc; + let $sync_slave_connection= server_3; + source include/sync_slave_sql_with_master.inc; + + --inc $j +} +--enable_query_log + +echo "Sleep for 70s to expire the logs"; +select sleep(70); + +echo "Inserting another row on leader: server_1. This will trigger auto purge on followers"; +connection server_1; +eval insert into t1 values($j); +select count(*) from t1; +--inc $j + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +connection server_2; +echo "Verifying apply logs on follower: server_2"; +select count(*) from t1; +source include/show_binary_logs.inc; + +connection server_3; +echo "Verifying apply logs on follower: server_3"; +select count(*) from t1; +source include/show_binary_logs.inc; + +connection server_1; +drop table t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart-master.opt b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart-master.opt new file mode 100644 index 000000000000..5c4b611c64fe --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart-master.opt @@ -0,0 +1,2 @@ +--loose-debug=+d,force_rotate --apply_log_retention_num=10 +--apply_log_retention_duration=1 diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart-slave.opt b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart-slave.opt new file mode 100644 index 000000000000..5c4b611c64fe --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart-slave.opt @@ -0,0 +1,2 @@ +--loose-debug=+d,force_rotate --apply_log_retention_num=10 +--apply_log_retention_duration=1 diff --git a/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart.test b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart.test new file mode 100644 index 000000000000..55eae66a5e74 --- /dev/null +++ b/mysql-test/suite/rpl_raft/t/rpl_raft_auto_purge_apply_logs_restart.test @@ -0,0 +1,89 @@ +source ../include/raft_3_node.inc; + +call mtr.add_suppression("Engine has seen trxs till file .*"); + +connection server_1; +show status like 'rpl_raft_role'; + +connection server_2; +show status like 'rpl_raft_role'; + +connection server_3; +show status like 'rpl_raft_role'; + +connection server_1; +create table t1 (a int primary key) engine=innodb; + +# Insert some rows into t1 +echo "Inserting rows into t1 on server_1"; +--let $j = 0 +--disable_query_log +while ($j <= 11) +{ + connection server_1; + eval insert into t1 values($j); + + let $sync_slave_connection= server_2; + --source include/sync_slave_sql_with_master.inc + let $sync_slave_connection= server_3; + --source include/sync_slave_sql_with_master.inc + + --inc $j +} +--enable_query_log +select count(*) from t1; + +connection server_2; +echo "Verifying apply logs on follower: server_2"; +select count(*) from t1; +source include/show_binary_logs.inc; + +connection server_3; +echo "Verifying apply logs on follower: server_3"; +select count(*) from t1; +source include/show_binary_logs.inc; + +connection server_2; +echo "Restarting follower: server_2"; +let $rpl_server_number = 2; +source include/rpl_restart_server.inc; +source include/start_slave_sql.inc; + +connection server_2; +echo "Verifying apply logs on follower: server_2"; +source include/show_binary_logs.inc; +select count(*) from t1; + +connection server_1; +echo "Sleep for 70s to expire apply logs on followers"; +select sleep(70); + +echo "Inserting one row in leader: server_1. This should trigger auto purge of apply logs on followers"; +eval insert into t1 values($j); +--inc $j +select count(*) from t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +connection server_2; +echo "Verifying apply logs on follower: server_2"; +select count(*) from t1; +source include/show_binary_logs.inc; + +connection server_3; +echo "Verifying apply logs on follower: server_3"; +select count(*) from t1; +source include/show_binary_logs.inc; + +connection server_1; +drop table t1; + +let $sync_slave_connection= server_2; +source include/sync_slave_sql_with_master.inc; +let $sync_slave_connection= server_3; +source include/sync_slave_sql_with_master.inc; + +source include/rpl_end.inc; diff --git a/mysql-test/suite/sys_vars/r/apply_log_retention_duration_basic.result b/mysql-test/suite/sys_vars/r/apply_log_retention_duration_basic.result new file mode 100644 index 000000000000..fd8df396b417 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/apply_log_retention_duration_basic.result @@ -0,0 +1,17 @@ +Default value of apply_log_retention_duration is 10 +SELECT @@global.apply_log_retention_duration; +@@global.apply_log_retention_duration +15 +SELECT @@session.apply_log_retention_duration; +ERROR HY000: Variable 'apply_log_retention_duration' is a GLOBAL variable +Expected error 'Variable is a GLOBAL variable' +apply_log_retention_duration is a dynamic variable +set @@global.apply_log_retention_duration = 17; +SELECT @@global.apply_log_retention_duration; +@@global.apply_log_retention_duration +17 +restore the default value +SET @@global.apply_log_retention_duration = default; +SELECT @@global.apply_log_retention_duration; +@@global.apply_log_retention_duration +15 diff --git a/mysql-test/suite/sys_vars/r/apply_log_retention_num_basic.result b/mysql-test/suite/sys_vars/r/apply_log_retention_num_basic.result new file mode 100644 index 000000000000..da8c63003ba6 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/apply_log_retention_num_basic.result @@ -0,0 +1,17 @@ +Default value of apply_log_retention_num is 15 +SELECT @@global.apply_log_retention_num; +@@global.apply_log_retention_num +10 +SELECT @@session.apply_log_retention_num; +ERROR HY000: Variable 'apply_log_retention_num' is a GLOBAL variable +Expected error 'Variable is a GLOBAL variable' +apply_log_retention_num is a dynamic variable +set @@global.apply_log_retention_num = 17; +SELECT @@global.apply_log_retention_num; +@@global.apply_log_retention_num +17 +restore the default value +SET @@global.apply_log_retention_num = default; +SELECT @@global.apply_log_retention_num; +@@global.apply_log_retention_num +10 diff --git a/mysql-test/suite/sys_vars/t/apply_log_retention_duration_basic.test b/mysql-test/suite/sys_vars/t/apply_log_retention_duration_basic.test new file mode 100644 index 000000000000..b582c67fd468 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/apply_log_retention_duration_basic.test @@ -0,0 +1,28 @@ +-- source include/load_sysvars.inc + +#### +# Verify the default value +#### +--echo Default value of apply_log_retention_duration is 10 +SELECT @@global.apply_log_retention_duration; + +#### +# Verify that this is not a session variable +#### +--Error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.apply_log_retention_duration; +--echo Expected error 'Variable is a GLOBAL variable' + +#### +## Verify that the variable is dynamic +#### +--echo apply_log_retention_duration is a dynamic variable +set @@global.apply_log_retention_duration = 17; +SELECT @@global.apply_log_retention_duration; + +#### +## Restore the default value +#### +--echo restore the default value +SET @@global.apply_log_retention_duration = default; +SELECT @@global.apply_log_retention_duration; diff --git a/mysql-test/suite/sys_vars/t/apply_log_retention_num_basic.test b/mysql-test/suite/sys_vars/t/apply_log_retention_num_basic.test new file mode 100644 index 000000000000..46d14a1e624e --- /dev/null +++ b/mysql-test/suite/sys_vars/t/apply_log_retention_num_basic.test @@ -0,0 +1,28 @@ +-- source include/load_sysvars.inc + +#### +# Verify the default value +#### +--echo Default value of apply_log_retention_num is 15 +SELECT @@global.apply_log_retention_num; + +#### +# Verify that this is not a session variable +#### +--Error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.apply_log_retention_num; +--echo Expected error 'Variable is a GLOBAL variable' + +#### +## Verify that the variable is dynamic +#### +--echo apply_log_retention_num is a dynamic variable +set @@global.apply_log_retention_num = 17; +SELECT @@global.apply_log_retention_num; + +#### +## Restore the default value +#### +--echo restore the default value +SET @@global.apply_log_retention_num = default; +SELECT @@global.apply_log_retention_num; diff --git a/sql/binlog.cc b/sql/binlog.cc index f182edc8ef64..6186547a54b1 100644 --- a/sql/binlog.cc +++ b/sql/binlog.cc @@ -3147,6 +3147,7 @@ MYSQL_BIN_LOG::MYSQL_BIN_LOG(uint *sync_period) memset(&index_file, 0, sizeof(index_file)); memset(&purge_index_file, 0, sizeof(purge_index_file)); memset(&crash_safe_index_file, 0, sizeof(crash_safe_index_file)); + apply_file_count.store(0); } @@ -3359,6 +3360,13 @@ int MYSQL_BIN_LOG::init_index_file() error= 1; } + if (!error && is_apply_log) + { + uint64_t num_apply_files= 0; + error= get_total_log_files(/*need_lock_index=*/false, &num_apply_files); + apply_file_count.store(num_apply_files); + } + return error; } @@ -3383,7 +3391,8 @@ int MYSQL_BIN_LOG::remove_deleted_logs_from_index(bool need_lock_index, bool need_update_threads) { int error; - int no_of_log_files_purged= 0; + uint64_t no_of_log_files_purged= 0; + uint64_t num_apply_files= 0; LOG_INFO log_info; DBUG_ENTER("remove_deleted_logs_from_index"); @@ -3412,9 +3421,26 @@ int MYSQL_BIN_LOG::remove_deleted_logs_from_index(bool need_lock_index, } if (no_of_log_files_purged) + { error= remove_logs_from_index(&log_info, need_update_threads); + if (is_apply_log) + { + // Fix number of apply log file count + if (apply_file_count >= no_of_log_files_purged) + apply_file_count -= no_of_log_files_purged; + else + { + error= get_total_log_files(need_lock_index, &num_apply_files); + apply_file_count.store(num_apply_files); - DBUG_PRINT("info",("num binlogs deleted = %d",no_of_log_files_purged)); + // NO_LINT_DEBUG + sql_print_information("Fixed apply file count (%lu) by reading from " + "index file.", apply_file_count.load()); + } + } + } + + DBUG_PRINT("info",("num binlogs deleted = %lu",no_of_log_files_purged)); err: if (need_lock_index) @@ -4685,6 +4711,10 @@ int MYSQL_BIN_LOG::add_log_to_index(uchar* log_name, { DBUG_SET("-d,simulate_no_free_space_error"); }); my_free(previous_gtid_set_buffer); + + if (enable_raft_plugin && is_apply_log) + apply_file_count++; + DBUG_RETURN(0); err: @@ -4792,6 +4822,34 @@ uint split_file_name_and_gtid_set_length(char *file_name_and_gtid_set_length) return 0; } +int MYSQL_BIN_LOG::get_total_log_files( + bool need_lock_index, uint64_t* num_log_files) +{ + int error= 0; + LOG_INFO temp_log_info; + *num_log_files= 0; + + if (need_lock_index) + mysql_mutex_lock(&LOCK_index); + + if ((error= find_log_pos( + &temp_log_info, /*log_name=*/NullS, /*need_lock_index=*/false))) + goto err; + + *num_log_files = *num_log_files + 1; + while (!(error= find_next_log(&temp_log_info, /*need_lock_index=*/false))) + *num_log_files = *num_log_files + 1; + +err: + if (need_lock_index) + mysql_mutex_unlock(&LOCK_index); + + if (error == LOG_INFO_EOF) + error= 0; // EOF is not an error + + return error; +} + /** Find the position in the log-index-file for the given log name. @@ -5384,6 +5442,26 @@ int MYSQL_BIN_LOG::purge_first_log(Relay_log_info* rli, bool included) } +void MYSQL_BIN_LOG::purge_apply_logs() +{ + if (!is_apply_log) + return; + + // No need to trigger purge if number of apply log files in the system + // currently is lower than apply_log_retention_num + if (apply_file_count <= apply_log_retention_num) + return; + + time_t purge_time = my_time(0) - apply_log_retention_duration /* mins */ * 60; + if (purge_time > 0) + { + ha_flush_logs(NULL); + purge_logs_before_date( + purge_time, /*auto_purge=*/true, /*stop_purge=*/true); + } + + return; +} /** Remove all logs before the given log from disk and from the index file. @@ -5421,6 +5499,7 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, int error= 0, error_index= 0, no_of_log_files_to_purge= 0, no_of_threads_locking_log= 0; + uint64_t num_log_files= 0; bool exit_loop= 0; bool found_last_safe_file= false; LOG_INFO log_info; @@ -5548,6 +5627,26 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, } } + if (enable_raft_plugin && is_apply_log) + { + if (apply_file_count > delete_list.size()) + apply_file_count -= delete_list.size(); + else + { + // NO_LINT_DEBUG + sql_print_information("Apply file count needs to be fixed. " + "apply_file_count = %lu, number of deleted apply files = %lu", + apply_file_count.load(), delete_list.size()); + + error= get_total_log_files(/*need_lock_index=*/false, &num_log_files); + + apply_file_count.store(num_log_files); + // NO_LINT_DEBUG + sql_print_information("Fixed apply file count (%lu) by reading from " + "index file.", apply_file_count.load()); + } + } + DBUG_EXECUTE_IF("crash_purge_non_critical_after_update_index", DBUG_SUICIDE();); // Update gtid_state->lost_gtids @@ -5900,6 +5999,8 @@ int MYSQL_BIN_LOG::purge_logs_in_list(std::list& delete_list, @param thd Thread pointer @param purge_time Delete all log files before given date. @param auto_purge True if this is an automatic purge. + @param stop_purge True if this is purge of apply logs and we have to stop + purging files based on apply_log_retention_num @note If any of the logs before the deleted one is in use, @@ -5913,19 +6014,31 @@ int MYSQL_BIN_LOG::purge_logs_in_list(std::list& delete_list, mysql_file_stat() or mysql_file_delete() */ -int MYSQL_BIN_LOG::purge_logs_before_date(time_t purge_time, bool auto_purge) +int MYSQL_BIN_LOG::purge_logs_before_date( + time_t purge_time, bool auto_purge, bool stop_purge) { - int error; - int no_of_threads_locking_log= 0, no_of_log_files_purged= 0; + int error= 0; + int no_of_threads_locking_log= 0; + uint64_t no_of_log_files_purged= 0; bool log_is_active= false, log_is_in_use= false; char to_log[FN_REFLEN], copy_log_in_use[FN_REFLEN]; LOG_INFO log_info; MY_STAT stat_area; THD *thd= current_thd; + uint64_t max_files_to_purge= ULONG_MAX; DBUG_ENTER("purge_logs_before_date"); mysql_mutex_lock(&LOCK_index); + + if (enable_raft_plugin && is_apply_log && stop_purge) + { + if (apply_file_count <= apply_log_retention_num) + goto err; + + max_files_to_purge= apply_file_count - apply_log_retention_num; + } + to_log[0]= 0; if ((error=find_log_pos(&log_info, NullS, false/*need_lock_index=false*/))) @@ -5987,6 +6100,11 @@ int MYSQL_BIN_LOG::purge_logs_before_date(time_t purge_time, bool auto_purge) else break; } + + if (enable_raft_plugin && is_apply_log && stop_purge && + (no_of_log_files_purged >= max_files_to_purge)) + break; + if (find_next_log(&log_info, false/*need_lock_index=false*/)) break; } @@ -6897,6 +7015,10 @@ void MYSQL_BIN_LOG::purge() purge_logs_before_date(purge_time, true); } } + + // Auto purge apply logs based on retention parameters + if (is_apply_log) + purge_apply_logs(); #endif } @@ -7001,6 +7123,8 @@ int binlog_change_to_apply() goto err; } + mysql_bin_log.is_apply_log= true; + /* Configures what object is used by the current log to store processed gtid(s). This is necessary in the MYSQL_BIN_LOG::MYSQL_BIN_LOG to @@ -7020,8 +7144,6 @@ int binlog_change_to_apply() goto err; } - mysql_bin_log.is_apply_log = true; - err: mysql_bin_log.unlock_index(); @@ -7143,6 +7265,7 @@ int binlog_change_to_binlog() // unset apply log, so that masters // ordered_commit understands this mysql_bin_log.is_apply_log = false; + mysql_bin_log.apply_file_count.store(0); err: diff --git a/sql/binlog.h b/sql/binlog.h index 875d9199f841..99084ca042a9 100644 --- a/sql/binlog.h +++ b/sql/binlog.h @@ -624,6 +624,10 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG */ bool is_apply_log= false; + // Number of files that are created and maintained in index files + std::atomic apply_file_count; + + /* This is relay log */ bool is_relay_log; @@ -843,6 +847,7 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG */ bool setup_flush_done; + int open(const char *opt_name) { return open_binlog(opt_name); } bool change_stage(THD *thd, Stage_manager::StageID stage, THD* queue, mysql_mutex_t *leave, mysql_mutex_t *enter); @@ -1066,10 +1071,12 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG @retval other Failure */ bool flush_and_sync(bool async, const bool force); + void purge_apply_logs(); int purge_logs(const char *to_log, bool included, bool need_lock_index, bool need_update_threads, std::atomic_ullong *decrease_log_space, bool auto_purge); - int purge_logs_before_date(time_t purge_time, bool auto_purge); + int purge_logs_before_date( + time_t purge_time, bool auto_purge, bool stop_purge= 0); int purge_first_log(Relay_log_info* rli, bool included); int set_crash_safe_index_file_name(const char *base_file_name); int open_crash_safe_index_file(); @@ -1098,6 +1105,17 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG int find_log_pos(LOG_INFO* linfo, const char* log_name, bool need_lock_index); int find_next_log(LOG_INFO* linfo, bool need_lock_index); + + /** + * Get the total number of log file entries in the index file + * + * @param need_lock_index - should we aquire LOCK_index + * @param num_log_files (out) - number of log file entries in the index file + * + * @return 0 on success, non-zero on failure + * + */ + int get_total_log_files(bool need_lock_index, uint64_t* num_log_files); int get_current_log(LOG_INFO* linfo, bool need_lock_log= true); /* This is called to find out the most recent binlog file diff --git a/sql/mysqld.cc b/sql/mysqld.cc index f5da4b9cda11..6a07f71e70fc 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -509,6 +509,8 @@ ulonglong max_tmp_disk_usage; bool enable_raft_plugin= 0; bool disable_raft_log_repointing= 0; ulong opt_raft_signal_async_dump_threads= 0; +ulonglong apply_log_retention_num= 0; +ulonglong apply_log_retention_duration= 0; /* write_control_level: * Global variable to control write throttling for short running queries and diff --git a/sql/mysqld.h b/sql/mysqld.h index 8f541548966e..de1b5e75c210 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -459,6 +459,8 @@ extern ulonglong maximum_hlc_drift_ns; extern bool enable_raft_plugin; extern bool disable_raft_log_repointing; extern ulong opt_raft_signal_async_dump_threads; +extern ulonglong apply_log_retention_num; +extern ulonglong apply_log_retention_duration; /* Enable query checksum validation for queries with a checksum sent */ extern my_bool enable_query_checksum; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 7c54e748d2b5..807c7ca16099 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -7370,3 +7370,14 @@ static Sys_var_enum Sys_raft_signal_async_dump_threads( raft_signal_async_dump_threads_options, DEFAULT(AFTER_CONSENSUS), NO_MUTEX_GUARD, NOT_IN_BINLOG); +static Sys_var_ulonglong Sys_apply_log_retention_num( + "apply_log_retention_num", + "Minimum number of apply logs that need to be retained.", + GLOBAL_VAR(apply_log_retention_num), CMD_LINE(OPT_ARG), + VALID_RANGE(0, ULONGLONG_MAX), DEFAULT(10), BLOCK_SIZE(1)); + +static Sys_var_ulonglong Sys_apply_log_retention_duration( + "apply_log_retention_duration", + "Minimum duration (mins) that apply logs need to be retained.", + GLOBAL_VAR(apply_log_retention_duration), CMD_LINE(OPT_ARG), + VALID_RANGE(0, ULONGLONG_MAX), DEFAULT(15), BLOCK_SIZE(1));