From b57f0c3e5f8af584ec751d29901afb3f00ad7efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Barbier?= Date: Tue, 15 Oct 2024 16:05:07 +0700 Subject: [PATCH 1/2] Simplify handling of favicons to creation only for now. --- src/projects/signals.py | 23 ------------------ src/projects/tasks/fetch_favicon.py | 37 +++++++++++++++++++---------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/src/projects/signals.py b/src/projects/signals.py index bf01a58..84f38fb 100644 --- a/src/projects/signals.py +++ b/src/projects/signals.py @@ -8,30 +8,7 @@ @receiver(post_save, sender=Project) def post_save_fetch_favicon(sender, instance, created, **kwargs): - favicon_need_refresh = False - - # Fetch the existing instance from the database if it exists - try: - old_instance = Project.objects.get(pk=instance.pk) - # Check if the favicon value has changed - if old_instance.favicon != instance.favicon: - instance.favicon_task_status = 'SUCCESS' - instance.favicon_last_edited = timezone.now() - except Project.DoesNotExist: - # If the object does not exist in the database, it's a new object - favicon_need_refresh = True - if created: - # If instance.pk is None, it's a new object - favicon_need_refresh = True - - if instance.favicon_task_status == 'UNKNOWN': - favicon_need_refresh = True - - if (timezone.now() - instance.favicon_last_edited).seconds > DELAY_REFRESH_FAVICON_SECONDS: - favicon_need_refresh = True - - if favicon_need_refresh: instance.favicon_task_status = 'PENDING' instance.favicon_last_edited = timezone.now() fetch_favicon.delay(instance.pk, instance.url) \ No newline at end of file diff --git a/src/projects/tasks/fetch_favicon.py b/src/projects/tasks/fetch_favicon.py index 9eb7852..f2f27f2 100644 --- a/src/projects/tasks/fetch_favicon.py +++ b/src/projects/tasks/fetch_favicon.py @@ -5,6 +5,8 @@ from urllib.parse import urljoin, urlparse from PIL import Image from io import BytesIO +from django.utils import timezone +from projects.models import Project @shared_task() def fetch_favicon(pk, url): @@ -64,23 +66,32 @@ def fetch_favicon(pk, url): if largest_favicon: print(f"Largest favicon found: {largest_favicon['url']} ({largest_favicon['width']}x{largest_favicon['height']})") - else: - print("No favicon found.") + # Fetch the content of the favicon + response = requests.get(largest_favicon['url']) + response.raise_for_status() # Ensure the request was successful + + # Wrap the content in a BytesIO object + favicon_content = BytesIO(response.content) - from projects.models import Project + # Get the Project instance + project = Project.objects.get(pk=pk) - # Fetch the content of the favicon - response = requests.get(largest_favicon['url']) - response.raise_for_status() # Ensure the request was successful + # Save the favicon to the project's ImageField or FileField + project.favicon.save(largest_favicon['url'].split('/')[-1], favicon_content) + else: + print("No favicon found.") + # Get the Project instance + project = Project.objects.get(pk=pk) + project.favicon_last_edited = timezone.now() + project.favicon_task_status = 'FAILED' + project.save() - # Wrap the content in a BytesIO object - favicon_content = BytesIO(response.content) + except requests.RequestException as e: + print(f"Error fetching the webpage: {e}") # Get the Project instance project = Project.objects.get(pk=pk) + project.favicon_last_edited = timezone.now() + project.favicon_task_status = 'FAILED' + project.save() - # Save the favicon to the project's ImageField or FileField - project.favicon.save(largest_favicon['url'].split('/')[-1], favicon_content) - - except requests.RequestException as e: - print(f"Error fetching the webpage: {e}") From ce76b448528c00d344472a3246b9fa8dd4dade7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Barbier?= Date: Tue, 15 Oct 2024 16:12:49 +0700 Subject: [PATCH 2/2] Make task update icon status --- src/projects/tasks/fetch_favicon.py | 80 ++++++++++++++------------- src/projects/tasks/refresh_favicon.py | 2 +- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/src/projects/tasks/fetch_favicon.py b/src/projects/tasks/fetch_favicon.py index f2f27f2..631026a 100644 --- a/src/projects/tasks/fetch_favicon.py +++ b/src/projects/tasks/fetch_favicon.py @@ -34,35 +34,36 @@ def fetch_favicon(pk, url): largest_size = (0, 0) # If one url in favicon end with svg we keep this ad largest favicon - if any(favicon_url.endswith('.svg') for favicon_url in favicons): - largest_favicon = { - 'url': next(favicon_url for favicon_url in favicons if favicon_url.endswith('.svg')), - 'width': 0, - 'height': 0, - } - print(f"Largest favicon found: {largest_favicon} (SVG)") - else: - for favicon_url in favicons: - try: - # Fetch the favicon image - favicon_response = requests.get(favicon_url, timeout=10) - favicon_response.raise_for_status() - - # Open the image and get its size - image = Image.open(BytesIO(favicon_response.content)) - width, height = image.size - - # Update the largest favicon if this one is larger - if width * height > largest_size[0] * largest_size[1]: - largest_size = (width, height) - largest_favicon = { - 'url': favicon_url, - 'width': width, - 'height': height, - } - - except Exception as e: - print(f"Error fetching or processing {favicon_url}: {e}") + for favicon_url in favicons: + try: + # Fetch the favicon image + favicon_response = requests.get(favicon_url, timeout=10) + favicon_response.raise_for_status() + + # Open the image and get its size + image = Image.open(BytesIO(favicon_response.content)) + width, height = image.size + + # Update the largest favicon if this one is larger + if width * height > largest_size[0] * largest_size[1]: + largest_size = (width, height) + largest_favicon = { + 'url': favicon_url, + 'width': width, + 'height': height, + } + + except Exception as e: + print(f"Error fetching or processing {favicon_url}: {e}") + + # If no image found, we try to get the first svg found + if not largest_favicon: + if any(favicon_url.endswith('.svg') for favicon_url in favicons): + largest_favicon = { + 'url': next(favicon_url for favicon_url in favicons if favicon_url.endswith('.svg')), + 'width': 0, + 'height': 0, + } if largest_favicon: print(f"Largest favicon found: {largest_favicon['url']} ({largest_favicon['width']}x{largest_favicon['height']})") @@ -75,23 +76,26 @@ def fetch_favicon(pk, url): # Get the Project instance project = Project.objects.get(pk=pk) - # Save the favicon to the project's ImageField or FileField project.favicon.save(largest_favicon['url'].split('/')[-1], favicon_content) + project.favicon_task_status = 'SUCCESS' + project.save() else: print("No favicon found.") # Get the Project instance project = Project.objects.get(pk=pk) project.favicon_last_edited = timezone.now() - project.favicon_task_status = 'FAILED' + project.favicon_task_status = 'FAILURE' project.save() - except requests.RequestException as e: + except e: print(f"Error fetching the webpage: {e}") - - # Get the Project instance - project = Project.objects.get(pk=pk) - project.favicon_last_edited = timezone.now() - project.favicon_task_status = 'FAILED' - project.save() + try: + # Get the Project instance + project = Project.objects.get(pk=pk) + project.favicon_last_edited = timezone.now() + project.favicon_task_status = 'FAILURE' + project.save() + except e: + print(f"Error editing the project: {e}") diff --git a/src/projects/tasks/refresh_favicon.py b/src/projects/tasks/refresh_favicon.py index ab5aee8..cf0544a 100644 --- a/src/projects/tasks/refresh_favicon.py +++ b/src/projects/tasks/refresh_favicon.py @@ -40,7 +40,7 @@ def revoke_tasks(tasks): projects = Project.objects.filter( favicon_last_edited__lt=six_hours_ago, ).exclude( - favicon_task_status='QUEUED' # Exclude 'QUEUED' status + favicon_task_status='PENDING' # Exclude 'PENDING' status ) print(f'Found {projects.count()} projects to refresh favicon.') for project in projects: