Major fixes to ingest-processor and cover-enforcer

root/app/calibre-web/cps/editbooks.py

@@ -1204,7 +1204,7 @@ def edit_cc_data(book_id, book, to_save, cc):
     # CWA Export of changed Metadata
     now = datetime.now()
     with open(f'/app/calibre-web-automated/metadata_change_logs/{now.strftime("%Y%m%d%H%M%S")}-{book_id}.json', 'w') as f:
-        json.dump(to_save, f)
+        json.dump(to_save, f, indent=4)
     return changed
 
 

root/etc/s6-overlay/s6-rc.d/cwa-ingest-service/run

@@ -12,7 +12,25 @@ echo "[cwa-ingest-service]: Watching folder: $WATCH_FOLDER"
 # Monitor the folder for new files
 s6-setuidgid abc inotifywait -m -r --format="%e %w%f" -e close_write -e moved_to "$WATCH_FOLDER" |
 while read -r events filepath ; do
+        # if [[ $(grep "$filepath" ingest-log-test.txt | egrep -o '[0-9]{10}') ]]; then
+        #         CURRENT_TIME=$(date +'%s')
+        #         TIME_OF_MATCH=$(grep "$filepath" ingest-log-test.txt | egrep -o '[0-9]{10}')
+        #         TODO NEED TO GET DIFFERENCE BETWEEN THE 2 TIMES AND IF LESS THAN 60 SECONDS, IGNORE
         echo "[cwa-ingest-service]: New files detected - $filepath - Starting Ingest Processor..."
-        python3 /app/calibre-web-automated/scripts/ingest_processor.py "$filepath"
+        python3 /app/calibre-web-automated/scripts/ingest_processor.py "$filepath" # &
+        # echo "'${filepath}' - $(date +'%s')" >> /config/.ingest_dupe_list
+        # INGEST_PROCESSOR_PID=$!
+        # Wait for the ingest processor to finish
+        # wait $INGEST_PROCESSOR_PID
+        # if ! [[ $(ls -A "$WATCH_FOLDER") ]]; then
+        #         FILES="${WATCH_FOLDER}/*"
+        #         for f in $FILES
+        #         do
+        #                 python3 /app/calibre-web-automated/scripts/ingest_processor.py "$f" &
+        #                 INGEST_PROCESSOR_PID=$!
+        #                 # Wait for the ingest processor to finish
+        #                 wait $INGEST_PROCESSOR_PID
+        #         done
+        # fi
 done
 

scripts/cover_enforcer.py

@@ -141,8 +141,8 @@ def read_log(self, auto=True, log_path: str = "None") -> dict:
         return log_info
 
     def get_book_dir_from_log(self, log_info: dict) -> str:
-        book_title = log_info['book_title'].replace(':', '_')
-        author_name = (log_info['author_name'].split(', ')[0]).split(' & ')[0]
+        book_title = log_info['title'].replace(':', '_')
+        author_name = (log_info['authors'].split(', ')[0]).split(' & ')[0]
         book_id = log_info['book_id']
 
         for char in book_title:
@@ -326,7 +326,7 @@ def main():
         if enforcer.enforcer_on:
             book_objects = enforcer.enforce_cover(book_dir)
             if not book_objects:
-                print(f"[cover-metadata-enforcer] Metadata for '{log_info['book_title']}' not successfully enforced")
+                print(f"[cover-metadata-enforcer] Metadata for '{log_info['title']}' not successfully enforced")
                 sys.exit(1)
             for book in book_objects:
                 book.log_info = log_info
@@ -335,7 +335,7 @@ def main():
             enforcer.delete_log()
             enforcer.check_for_other_logs()
         else: # Enforcer has been disabled in the CWA Settings
-            print(f"[cover-metadata-enforcer] The CWA Automatic Metadata enforcement service is currently disabled in the settings. Therefore the metadata changes for {log_info['book_title'].replace(':', '_')} won't be enforced.\n\nThis means that the changes made will appear in the Web UI, but not be stored in the ebook files themselves.")
+            print(f"[cover-metadata-enforcer] The CWA Automatic Metadata enforcement service is currently disabled in the settings. Therefore the metadata changes for {log_info['title'].replace(':', '_')} won't be enforced.\n\nThis means that the changes made will appear in the Web UI, but not be stored in the ebook files themselves.")
             enforcer.delete_log()
     else:
         parser.print_usage()

scripts/cwa_db.py

@@ -239,7 +239,7 @@ def update_cwa_settings(self, result) -> None:
 
     def enforce_add_entry_from_log(self, log_info: dict):
         """Adds an entry to the db from a change log file"""
-        self.cur.execute("INSERT INTO cwa_enforcement(timestamp, book_id, book_title, author, file_path, trigger_type) VALUES (?, ?, ?, ?, ?, ?);", (log_info['timestamp'], log_info['book_id'], log_info['book_title'], log_info['author_name'], log_info['file_path'], 'auto -log'))
+        self.cur.execute("INSERT INTO cwa_enforcement(timestamp, book_id, book_title, author, file_path, trigger_type) VALUES (?, ?, ?, ?, ?, ?);", (log_info['timestamp'], log_info['book_id'], log_info['title'], log_info['authors'], log_info['file_path'], 'auto -log'))
         self.con.commit()
 
 

scripts/ingest_processor.py

@@ -151,6 +151,8 @@ def convert_to_kepub(self) -> tuple[bool,str]:
                 print(f"[ingest-processor]: CON_ERROR: {self.filename} could not be converted to kepub due to the following error:\nEXIT/ERROR CODE: {e.returncode}\n{e.stderr}", flush=True)
                 shutil.copy2(converted_filepath, f"/config/processed_books/failed/{os.path.basename(self.filepath)}")
                 return False, ""
+            except Exception as e:
+                print(f"[ingest-processor] ingest-processor ran into the following error:\n{e}", flush=True)
         else:
             print(f"[ingest-processor]: An error occurred when converting the original {self.input_format} to epub. Cancelling kepub conversion...", flush=True)
             return False, ""
@@ -159,12 +161,15 @@ def convert_to_kepub(self) -> tuple[bool,str]:
     def delete_current_file(self) -> None:
         """Deletes file just processed from ingest folder"""
         os.remove(self.filepath) # Removes processed file
-        subprocess.run(["find", f"{self.ingest_folder}", "-type", "d", "-empty", "-delete"]) # Removes any now empty folders
+        subprocess.run(["find", f"{self.ingest_folder}", "-mindepth", "1", "-type", "d", "-empty", "-delete"]) # Removes any now empty folders in the ingest folder
 
 
     def add_book_to_library(self, book_path:str) -> None:
         if self.target_format == "epub" and self.is_kindle_epub_fixer:
-            self.kindle_epub_fixer(book_path)
+            self.run_kindle_epub_fixer(book_path, dest=self.tmp_conversion_dir)
+            fixed_epub_path = str(self.empty_tmp_con_dir) + str(os.path.basename(book_path))
+            if Path(fixed_epub_path).exists():
+                book_path = self.empty_tmp_con_dir + os.path.basename(book_path)
 
         print("[ingest-processor]: Importing new book to CWA...")
         import_path = Path(book_path)
@@ -182,11 +187,13 @@ def add_book_to_library(self, book_path:str) -> None:
         except subprocess.CalledProcessError as e:
             print(f"[ingest-processor] {import_path.stem} was not able to be added to the Calibre Library due to the following error:\nCALIBREDB EXIT/ERROR CODE: {e.returncode}\n{e.stderr}", flush=True)
             shutil.copy2(book_path, f"/config/processed_books/failed/{import_filename}")
+        except Exception as e:
+            print(f"[ingest-processor] ingest-processor ran into the following error:\n{e}", flush=True)
 
 
-    def kindle_epub_fixer(self, filepath:str) -> None:
+    def run_kindle_epub_fixer(self, filepath:str, dest=None) -> None:
         try:
-            EPUBFixer(filepath).process()
+            EPUBFixer(filepath, dest).process()
         except Exception as e:
             print(f"[ingest-processor] An error occurred while processing {os.path.basename(filepath)} with the kindle-epub-fixer. See the following error:\n{e}")
 
@@ -211,11 +218,12 @@ def set_library_permissions(self):
 def main(filepath=sys.argv[1]):
     """Checks if filepath is a directory. If it is, main will be ran on every file in the given directory
     Inotifywait won't detect files inside folders if the folder was moved rather than copied"""
-    if os.path.isdir(filepath):
-        print(os.listdir(filepath))
+    if os.path.isdir(filepath) and Path(filepath).exists():
+        # print(os.listdir(filepath))
         for filename in os.listdir(filepath):
             f = os.path.join(filepath, filename)
-            main(f)
+            if Path(f).exists():
+                main(f)
         return
 
     nbp = NewBookProcessor(filepath)

scripts/kindle_epub_fixer.py

@@ -8,8 +8,9 @@
 ### & modified and integrated into CWA by CrocodileStick
 
 class EPUBFixer:
-    def __init__(self, epub_path):
+    def __init__(self, epub_path:str, dest=None):
         self.epub_path = epub_path
+        self.dest = dest
         self.files = {}
         self.fixed_problems = []