Merge pull request #24 from wikitongues/bug-fix

Various bug fixes

main.py

@@ -7,9 +7,11 @@
 import luigi
 import luigi.execution_summary
 
+from tasks.archival_task import ArchivalTask
 from tasks.check_archival_status import CheckArchivalStatus
 from tasks.constants import ELIGIBILITY_FIELD
-from tasks.enums import Eligibility
+from tasks.enums import ArchivalStatus, Eligibility
+from tasks.exceptions import NoDropboxFolder, NoThumbnail, NoVideo
 from tasks.utils import get_airtable_client
 
 
@@ -28,6 +30,22 @@ def get_compliant_oh_id(oh_id: str) -> str:
     return ascii
 
 
+@ArchivalTask.event_handler(luigi.Event.FAILURE)
+def report_failure(task: ArchivalTask, exception: Exception):
+    if type(exception) == NoThumbnail:
+        task.airtable_client.update(
+            task.airtable_record_id, {task.status_field: ArchivalStatus.INVALID_THUMBNAIL.value}
+        )
+    elif type(exception) == NoVideo:
+        task.airtable_client.update(task.airtable_record_id, {task.status_field: ArchivalStatus.INVALID_VIDEO.value})
+    elif type(exception) == NoDropboxFolder:
+        task.airtable_client.update(
+            task.airtable_record_id, {task.status_field: ArchivalStatus.NO_DROPBOX_FOLDER.value}
+        )
+    else:
+        task.airtable_client.update(task.airtable_record_id, {task.status_field: ArchivalStatus.PROCESSING_ERROR.value})
+
+
 def run(id: str, dev: bool) -> bool:
     airtable = get_airtable_client()
     record = airtable.get(id, cell_format="string", time_zone="America/New_York", user_locale="en-ca")

requirements.txt

@@ -11,7 +11,7 @@ identify==2.6.0
 idna==3.8
 inflection==0.5.1
 lockfile==0.12.2
-luigi==3.5.1
+luigi @ git+https://github.com/smrohrer/luigi.git@fe2157aa08f2ce91e914c20b805338aa222fb93c
 nodeenv==1.9.1
 platformdirs==4.2.2
 ply==3.11

tasks/download.py

@@ -8,7 +8,6 @@
 
 from .archival_task import ArchivalTask
 from .constants import VALID_VIDEO_EXTENSIONS
-from .enums import ArchivalStatus
 from .exceptions import NoDropboxFolder, NoThumbnail, NoVideo
 
 
@@ -102,21 +101,11 @@ def validate(self) -> None:
             raise NoDropboxFolder
 
         if not self.has_valid_video(filenames):
-            self.logger.warn(f"No video found on Dropbox for {self.oh_id}; skipping.")
-
-            self.airtable_client.update(
-                self.airtable_record_id, {self.status_field: ArchivalStatus.INVALID_VIDEO.value}
-            )
-
+            self.logger.warning(f"No video found on Dropbox for {self.oh_id}; skipping.")
             raise NoVideo
 
         if not self.has_valid_thumbnail(filenames):
-            self.logger.warn(f"No thumbnail found on Dropbox for {self.oh_id}; skipping.")
-
-            self.airtable_client.update(
-                self.airtable_record_id, {self.status_field: ArchivalStatus.INVALID_THUMBNAIL.value}
-            )
-
+            self.logger.warning(f"No thumbnail found on Dropbox for {self.oh_id}; skipping.")
             raise NoThumbnail
 
     def download(self):

tasks/enums.py

@@ -10,4 +10,5 @@ class ArchivalStatus(Enum):
     ARCHIVED = "Archived"
     INVALID_THUMBNAIL = "Invalid Thumbnail"
     INVALID_VIDEO = "Invalid Video"
+    NO_DROPBOX_FOLDER = "No Dropbox Folder"
     PROCESSING_ERROR = "Processing Error"

tasks/prepare.py

@@ -86,7 +86,7 @@ def use_raw_thumbnail(self):
         # Is there any jpg file in the root, perhaps with a legacy filename?
         root_jpgs = list(
             filter(
-                lambda filename: re.match(rf"^{re.escape(str(path))}\/[\w_]+\.jpg$", filename, re.IGNORECASE),
+                lambda filename: re.match(rf"^{re.escape(str(path))}\/[^\/]+\.jpg$", filename, re.IGNORECASE),
                 self.output().fs.listdir(str(path)),
             )
         )

tasks/upload.py

@@ -19,9 +19,9 @@
 
 
 class UploadTarget(luigi.contrib.dropbox.DropboxTarget):
-    def __init__(self, staging_dir, compliant_oh_id, token):
+    def __init__(self, staging_dir, compliant_oh_id, token, root_namespace_id):
         path = f"{staging_dir}/{compliant_oh_id}"
-        super().__init__(path, token)
+        super().__init__(path, token, root_namespace_id=root_namespace_id)
         self.compliant_oh_id = compliant_oh_id
 
     def exists(self):
@@ -71,11 +71,15 @@ def requires(self):
         return Stage(**self.param_kwargs)
 
     def output(self):
-        return UploadTarget(self.dropbox_staging_dir, self.compliant_oh_id, self.dropbox_token)
+        return UploadTarget(
+            self.dropbox_staging_dir, self.compliant_oh_id, self.dropbox_token, self.dropbox_root_namespace_id
+        )
 
     @cached_property
     def dbx(self):
-        return dropbox.Dropbox(self.dropbox_token, timeout=TIMEOUT)
+        return dropbox.Dropbox(self.dropbox_token, timeout=TIMEOUT).with_path_root(
+            dropbox.common.PathRoot.root(self.dropbox_root_namespace_id)
+        )
 
     def upload_file(self, file_path: str) -> None:
         target_path = file_path.replace(self.local_staging_dir, self.dropbox_staging_dir)