From 719cbb0e7981e627de798d71b6fc95c15cb3d6fc Mon Sep 17 00:00:00 2001 From: Cordila <49218334+Cordila@users.noreply.github.com> Date: Mon, 13 Mar 2023 13:38:41 +0530 Subject: [PATCH 1/5] Add config to change how long logs are stored --- bot.py | 42 ++++++++++++++++++++++++++++++++++++++---- cogs/plugins.py | 2 -- cogs/utility.py | 1 - core/config.py | 4 ++-- core/config_help.json | 11 +++++++++++ core/models.py | 1 - core/thread.py | 2 -- 7 files changed, 51 insertions(+), 12 deletions(-) diff --git a/bot.py b/bot.py index b23b2449b2..5bfe640f49 100644 --- a/bot.py +++ b/bot.py @@ -12,7 +12,7 @@ import sys import platform import typing -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from subprocess import PIPE from types import SimpleNamespace @@ -633,6 +633,7 @@ async def on_ready(self): self.post_metadata.start() self.autoupdate.start() + self.log_expiry.start() self._started = True async def convert_emoji(self, name: str) -> str: @@ -657,7 +658,6 @@ async def get_or_fetch_user(self, id: int) -> discord.User: return self.get_user(id) or await self.fetch_user(id) async def retrieve_emoji(self) -> typing.Tuple[str, str]: - sent_emoji = self.config["sent_emoji"] blocked_emoji = self.config["blocked_emoji"] @@ -731,7 +731,6 @@ def check_manual_blocked_roles(self, author: discord.Member) -> bool: if isinstance(author, discord.Member): for r in author.roles: if str(r.id) in self.blocked_roles: - blocked_reason = self.blocked_roles.get(str(r.id)) or "" try: @@ -790,7 +789,6 @@ async def is_blocked( channel: discord.TextChannel = None, send_message: bool = False, ) -> bool: - member = self.guild.get_member(author.id) if member is None: # try to find in other guilds @@ -1714,6 +1712,42 @@ async def before_autoupdate(self): self.autoupdate.cancel() return + @tasks.loop() + async def log_expiry(self): + log_expire_after = self.config.get("log_expiration") + if log_expire_after == isodate.Duration(): + return self.log_expiry.stop() + + now = datetime.utcnow() + expiration_datetime = now - log_expire_after + expired_logs = await self.db.logs.find({"closed_at": {"$lte": str(expiration_datetime)}}).to_list( + None + ) + + if not await self.db.logs.find_one(): + return await discord.utils.sleep_until(now + timedelta(days=1)) + + if not expired_logs: + fetch = await self.db.logs.find().sort("closed_at", 1).limit(1).to_list(None) + if fetch: + for x in fetch: + if x["closed_at"] > str(expiration_datetime): + closed_at = datetime.strptime(x["closed_at"], "%Y-%m-%d %H:%M:%S.%f%z") + next_expiry = closed_at + log_expire_after + return await discord.utils.sleep_until(next_expiry) + + for log in expired_logs: + await self.db.logs.delete_one({"_id": log["_id"]}) + logger.info(f"Deleted {len(expired_logs)} expired logs.") + + fetch = await self.db.logs.find().sort("closed_at", 1).limit(1).to_list(None) + if fetch: + for x in fetch: + if x["closed_at"] > str(expiration_datetime): + closed_at = datetime.strptime(x["closed_at"], "%Y-%m-%d %H:%M:%S.%f%z") + next_expiry = closed_at + log_expire_after + return await discord.utils.sleep_until(next_expiry) + def format_channel_name(self, author, exclude_channel=None, force_null=False): """Sanitises a username for use with text channel names diff --git a/cogs/plugins.py b/cogs/plugins.py index 2bfac509af..37020f7e0c 100644 --- a/cogs/plugins.py +++ b/cogs/plugins.py @@ -265,7 +265,6 @@ async def load_plugin(self, plugin): raise InvalidPluginError("Cannot load extension, plugin invalid.") from exc async def parse_user_input(self, ctx, plugin_name, check_version=False): - if not self.bot.config["enable_plugins"]: embed = discord.Embed( description="Plugins are disabled, enable them by setting `ENABLE_PLUGINS=true`", @@ -380,7 +379,6 @@ async def plugins_add(self, ctx, *, plugin_name: str): await self.bot.config.update() if self.bot.config.get("enable_plugins"): - invalidate_caches() try: diff --git a/cogs/utility.py b/cogs/utility.py index ac642d9eb3..26356289df 100644 --- a/cogs/utility.py +++ b/cogs/utility.py @@ -599,7 +599,6 @@ async def status(self, ctx, *, status_type: str.lower): return await ctx.send(embed=embed) async def set_presence(self, *, status=None, activity_type=None, activity_message=None): - if status is None: status = self.bot.config.get("status") diff --git a/core/config.py b/core/config.py index 56db40c7b0..a264d911ba 100644 --- a/core/config.py +++ b/core/config.py @@ -21,7 +21,6 @@ class ConfigManager: - public_keys = { # activity "twitch_url": "https://www.twitch.tv/discordmodmail/", @@ -37,6 +36,7 @@ class ConfigManager: "account_age": isodate.Duration(), "guild_age": isodate.Duration(), "thread_cooldown": isodate.Duration(), + "log_expiration": isodate.Duration(), "reply_without_command": False, "anon_reply_without_command": False, "plain_reply_without_command": False, @@ -183,7 +183,7 @@ class ConfigManager: colors = {"mod_color", "recipient_color", "main_color", "error_color"} - time_deltas = {"account_age", "guild_age", "thread_auto_close", "thread_cooldown"} + time_deltas = {"account_age", "guild_age", "thread_auto_close", "thread_cooldown", "log_expiration"} booleans = { "use_user_id_channel_name", diff --git a/core/config_help.json b/core/config_help.json index e7ebb9590d..4128983241 100644 --- a/core/config_help.json +++ b/core/config_help.json @@ -373,6 +373,17 @@ "To disable thread cooldown, do `{prefix}config del thread_cooldown`." ] }, + "log_expiration": { + "default": "Never", + "description": "Specify the duration of how long the thread log will be kept in the log channel. After the duration has passed, the log will be deleted.", + "examples": [ + "`{prefix}config set thread_cooldown P12DT3H` (stands for 12 days and 3 hours in [ISO-8601 Duration Format](https://en.wikipedia.org/wiki/ISO_8601#Durations))", + "`{prefix}config set thread_cooldown 3 days and 5 hours` (accepted readable time)" + ], + "notes": [ + "To disable log expiration, do `{prefix}config del log_expiration`." + ] + }, "thread_cancelled": { "default": "\"Cancelled\"", "description": "This is the message to display when a thread times out and creation is cancelled.", diff --git a/core/models.py b/core/models.py index 2eab1ceebb..4c6f956e82 100644 --- a/core/models.py +++ b/core/models.py @@ -202,7 +202,6 @@ async def convert(self, ctx, argument): try: return await super().convert(ctx, argument) except commands.ChannelNotFound: - if guild: categories = {c.name.casefold(): c for c in guild.categories} else: diff --git a/core/thread.py b/core/thread.py index 53cdd1d202..98756c46f5 100644 --- a/core/thread.py +++ b/core/thread.py @@ -716,7 +716,6 @@ async def delete_message( async def find_linked_message_from_dm( self, message, either_direction=False, get_thread_channel=False ) -> typing.List[discord.Message]: - joint_id = None if either_direction: joint_id = get_joint_id(message) @@ -909,7 +908,6 @@ async def send( persistent_note: bool = False, thread_creation: bool = False, ) -> None: - if not note and from_mod: self.bot.loop.create_task(self._restart_close_timer()) # Start or restart thread auto close From c34a9ed94760b880746135f30ca6def0c9194859 Mon Sep 17 00:00:00 2001 From: Cordila <49218334+Cordila@users.noreply.github.com> Date: Tue, 14 Mar 2023 08:42:09 +0530 Subject: [PATCH 2/5] Optimize deleting logs --- bot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot.py b/bot.py index 5bfe640f49..78cd63eb85 100644 --- a/bot.py +++ b/bot.py @@ -1736,8 +1736,7 @@ async def log_expiry(self): next_expiry = closed_at + log_expire_after return await discord.utils.sleep_until(next_expiry) - for log in expired_logs: - await self.db.logs.delete_one({"_id": log["_id"]}) + await self.db.logs.delete_many({"closed_at": {"$lte": str(expiration_datetime)}}) logger.info(f"Deleted {len(expired_logs)} expired logs.") fetch = await self.db.logs.find().sort("closed_at", 1).limit(1).to_list(None) From f6d737994be9262a9377745ac363fd0fa7630f18 Mon Sep 17 00:00:00 2001 From: Cordila <49218334+Cordila@users.noreply.github.com> Date: Tue, 14 Mar 2023 15:18:51 +0530 Subject: [PATCH 3/5] Update core/config_help.json Co-authored-by: khakers <22665282+khakers@users.noreply.github.com> Signed-off-by: Cordila <49218334+Cordila@users.noreply.github.com> --- core/config_help.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config_help.json b/core/config_help.json index 4128983241..29f3f27693 100644 --- a/core/config_help.json +++ b/core/config_help.json @@ -375,7 +375,7 @@ }, "log_expiration": { "default": "Never", - "description": "Specify the duration of how long the thread log will be kept in the log channel. After the duration has passed, the log will be deleted.", + "description": "The duration closed threads will be stored within the database before deletion. Logs that have been closed for longer than this duration will be deleted automatically.", "examples": [ "`{prefix}config set thread_cooldown P12DT3H` (stands for 12 days and 3 hours in [ISO-8601 Duration Format](https://en.wikipedia.org/wiki/ISO_8601#Durations))", "`{prefix}config set thread_cooldown 3 days and 5 hours` (accepted readable time)" From 2e439b57820802bf9112944cef51ddfa66b578db Mon Sep 17 00:00:00 2001 From: Cordila <49218334+Cordila@users.noreply.github.com> Date: Fri, 8 Sep 2023 08:55:24 +0530 Subject: [PATCH 4/5] Testing changes --- bot.py | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/bot.py b/bot.py index a0c67dcb30..dc949dd700 100644 --- a/bot.py +++ b/bot.py @@ -1712,40 +1712,17 @@ async def before_autoupdate(self): self.autoupdate.cancel() return - @tasks.loop() + @tasks.loop(hours=1, reconnect=False) async def log_expiry(self): log_expire_after = self.config.get("log_expiration") if log_expire_after == isodate.Duration(): return self.log_expiry.stop() - now = datetime.utcnow() + now = discord.utils.utcnow() expiration_datetime = now - log_expire_after - expired_logs = await self.db.logs.find({"closed_at": {"$lte": str(expiration_datetime)}}).to_list( - None - ) + expired_logs = await self.db.logs.delete_many({"closed_at": {"$lte": str(expiration_datetime)}}) - if not await self.db.logs.find_one(): - return await discord.utils.sleep_until(now + timedelta(days=1)) - - if not expired_logs: - fetch = await self.db.logs.find().sort("closed_at", 1).limit(1).to_list(None) - if fetch: - for x in fetch: - if x["closed_at"] > str(expiration_datetime): - closed_at = datetime.strptime(x["closed_at"], "%Y-%m-%d %H:%M:%S.%f%z") - next_expiry = closed_at + log_expire_after - return await discord.utils.sleep_until(next_expiry) - - await self.db.logs.delete_many({"closed_at": {"$lte": str(expiration_datetime)}}) - logger.info(f"Deleted {len(expired_logs)} expired logs.") - - fetch = await self.db.logs.find().sort("closed_at", 1).limit(1).to_list(None) - if fetch: - for x in fetch: - if x["closed_at"] > str(expiration_datetime): - closed_at = datetime.strptime(x["closed_at"], "%Y-%m-%d %H:%M:%S.%f%z") - next_expiry = closed_at + log_expire_after - return await discord.utils.sleep_until(next_expiry) + logger.info(f"Deleted {expired_logs.deleted_count} expired logs.") def format_channel_name(self, author, exclude_channel=None, force_null=False): """Sanitises a username for use with text channel names From ae6aa1df6ab09323bc109306031660c7d4942aed Mon Sep 17 00:00:00 2001 From: Cordila <49218334+Cordila@users.noreply.github.com> Date: Fri, 8 Sep 2023 13:32:37 +0530 Subject: [PATCH 5/5] Update core/config_help.json (Fix copy paste) Co-authored-by: Jerrie <70805800+Jerrie-Aries@users.noreply.github.com> Signed-off-by: Cordila <49218334+Cordila@users.noreply.github.com> --- core/config_help.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/config_help.json b/core/config_help.json index 575c5c4153..fe7c0c513f 100644 --- a/core/config_help.json +++ b/core/config_help.json @@ -377,8 +377,8 @@ "default": "Never", "description": "The duration closed threads will be stored within the database before deletion. Logs that have been closed for longer than this duration will be deleted automatically.", "examples": [ - "`{prefix}config set thread_cooldown P12DT3H` (stands for 12 days and 3 hours in [ISO-8601 Duration Format](https://en.wikipedia.org/wiki/ISO_8601#Durations))", - "`{prefix}config set thread_cooldown 3 days and 5 hours` (accepted readable time)" + "`{prefix}config set log_expiration P12DT3H` (stands for 12 days and 3 hours in [ISO-8601 Duration Format](https://en.wikipedia.org/wiki/ISO_8601#Durations))", + "`{prefix}config set log_expiration 3 days and 5 hours` (accepted readable time)" ], "notes": [ "To disable log expiration, do `{prefix}config del log_expiration`."