Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move mailboxes code to separate app #295

Open
wants to merge 38 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
75e48cd
Remove unused mail service functions
kevincarrogan Dec 20, 2024
99e116d
Rename test file to match file under test
kevincarrogan Dec 20, 2024
b53bc4a
Use context manager for managing pop3 connection
kevincarrogan Dec 21, 2024
5b35c4c
Remove default settings for mail server initialisation
kevincarrogan Dec 21, 2024
5431d0a
Allow configuration of mailservers to be defined in config
kevincarrogan Dec 21, 2024
2cb3741
Fix management command to check mailserver connections
kevincarrogan Dec 21, 2024
3ad7b34
Move mailboxes code to separate app
kevincarrogan Dec 22, 2024
f19405f
Add str methods for mailbox models
kevincarrogan Dec 22, 2024
2c32798
Move get_message_id function to mailboxes
kevincarrogan Dec 22, 2024
8901195
Extract out utils for getting headers and ids from emails
kevincarrogan Dec 22, 2024
4588b5e
Change API for mailbox functions to separate out responsibilities
kevincarrogan Dec 22, 2024
6da0cd4
Pull out utility for getting message number
kevincarrogan Dec 23, 2024
87de06c
Move getting read message function into utils
kevincarrogan Dec 23, 2024
994ab1e
Return a set of message ids
kevincarrogan Dec 23, 2024
d3cf6b3
Set reverse relation name for mail read status
kevincarrogan Dec 23, 2024
73740b3
Jump out of email loop earlier if message id has already been read
kevincarrogan Dec 23, 2024
1822cac
Split out getting unread messages into separate function
kevincarrogan Dec 23, 2024
e2c2b8d
Move out message iterator function
kevincarrogan Dec 23, 2024
d46f598
Rewrite getting message iterator
kevincarrogan Dec 23, 2024
fb747f2
Save email data to object
kevincarrogan Dec 23, 2024
8a2a4e9
Ensure message iterator runs with active pop connection
kevincarrogan Dec 24, 2024
a40ceef
Update to correctly match output from real mail servers
kevincarrogan Dec 24, 2024
428dc5d
Handle cases where the payload may have slightly changed when getting…
kevincarrogan Dec 24, 2024
8236a8b
By default only run unit tests
kevincarrogan Dec 24, 2024
444a1d7
Add integration tests
kevincarrogan Dec 24, 2024
401efa8
Split out tests into separate jobs
kevincarrogan Dec 24, 2024
afccf54
Add integration tests to circleci
kevincarrogan Dec 24, 2024
6b8252c
Add integration tests for message iterator
kevincarrogan Dec 24, 2024
c5be73e
Set better default value for database url
kevincarrogan Dec 24, 2024
ad95ace
Set distinct URL values in env and docker compose
kevincarrogan Dec 24, 2024
6353d46
Use docker.env file when running integration tests
kevincarrogan Dec 24, 2024
3551c71
Run all tests locally and ensure circleci doesn't run integration tes…
kevincarrogan Dec 31, 2024
1b51caa
Remove unused mock_hmrc app
kevincarrogan Dec 31, 2024
310693f
Remove untested and unused management command
kevincarrogan Dec 31, 2024
3fce8fb
Add tests for find_mail_of
kevincarrogan Dec 31, 2024
efb1856
Swap out mocking mailservers for overriding settings
kevincarrogan Dec 31, 2024
6b2ac82
Add comment for update_or_create on read statuses
kevincarrogan Jan 7, 2025
8135a5d
Add code comments and annotations to mailbox utils
kevincarrogan Jan 7, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 55 additions & 52 deletions mail/libraries/mailbox_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from mail.libraries.helpers import to_mail_message_dto
from mail.models import Mail
from mailboxes.enums import MailReadStatuses
from mailboxes.models import MailboxConfig, MailReadStatus
from mailboxes.models import MailboxConfig
from mailboxes.utils import (
get_message_header,
get_message_id,
Expand Down Expand Up @@ -52,60 +52,63 @@ def get_message_iterator(pop3_connection: POP3_SSL, username: str) -> Iterator[T
logger.info("Number of messages READ/UNPROCESSABLE in %s are %s", mailbox_config.username, len(read_messages))

for message_id, message_num in mail_message_ids:
if message_id in read_messages:
continue

# only return messages we haven't seen before
if message_id not in read_messages:
read_status, _ = MailReadStatus.objects.get_or_create(
message_id=message_id, message_num=message_num, mailbox=mailbox_config
read_status, _ = mailbox_config.mail_read_statuses.get_or_create(
message_id=message_id,
message_num=message_num,
)

def mark_status(status):
"""
:param status: A choice from `MailReadStatuses.choices`
"""
logger.info(
"Marking message_id %s with message_num %s from %s as %s",
message_id,
message_num,
read_status.mailbox,
status,
)
read_status.status = status
read_status.save()
currycoder marked this conversation as resolved.
Show resolved Hide resolved

try:
m = pop3_connection.retr(message_num)
logger.info(
"Retrieved message_id %s with message_num %s from %s",
message_id,
message_num,
read_status.mailbox,
)
except error_proto as err:
logger.error(
"Unable to RETR message num %s with Message-ID %s in %s: %s",
message_num,
message_id,
mailbox_config,
err,
exc_info=True,
)
continue

try:
mail_message = to_mail_message_dto(m)
except ValueError as ve:
logger.error(
"Unable to convert message num %s with Message-Id %s to DTO in %s: %s",
message_num,
message_id,
mailbox_config,
ve,
exc_info=True,
)
mark_status(MailReadStatuses.UNPROCESSABLE)
continue

def mark_status(status):
"""
:param status: A choice from `MailReadStatuses.choices`
"""
logger.info(
"Marking message_id %s with message_num %s from %s as %s",
message_id,
message_num,
read_status.mailbox.username,
status,
)
read_status.status = status
read_status.save()

try:
m = pop3_connection.retr(message_num)
logger.info(
"Retrieved message_id %s with message_num %s from %s",
message_id,
message_num,
read_status.mailbox.username,
)
except error_proto as err:
logger.error(
"Unable to RETR message num %s with Message-ID %s in %s: %s",
message_num,
message_id,
mailbox_config,
err,
exc_info=True,
)
continue

try:
mail_message = to_mail_message_dto(m)
except ValueError as ve:
logger.error(
"Unable to convert message num %s with Message-Id %s to DTO in %s: %s",
message_num,
message_id,
mailbox_config,
ve,
exc_info=True,
)
mark_status(MailReadStatuses.UNPROCESSABLE)
continue

yield mail_message, mark_status
yield mail_message, mark_status


def find_mail_of(extract_types: List[str], reception_status: str) -> Mail or None:
Expand Down