Add writing and updating JSON database

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
         language: system
         types: [text]
         stages: [commit, push, manual]
-        exclude_types: [html]
+        exclude_types: [html, json]
       - id: flake8
         name: flake8
         entry: flake8
@@ -54,9 +54,9 @@ repos:
         language: system
         types: [text]
         stages: [commit, push, manual]
-        exclude_types: [html]
+        exclude_types: [html, json]
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.3.0
     hooks:
       - id: prettier
-        exclude_types: [html]
+        exclude_types: [html, json]

pyproject.toml

@@ -50,6 +50,7 @@ Pygments = "^2.9.0"
 requests-mock = "^1.9.3"
 flake8-black = "^0.2.3"
 flake8-import-order = "^0.18.1"
+rope = "^0.20.1"
 
 [tool.poetry.scripts]
 lep-downloader = "lep_downloader.__main__:main"
@@ -63,7 +64,7 @@ source = ["lep_downloader"]
 
 [tool.coverage.report]
 show_missing = true
-fail_under = 85
+fail_under = 100
 
 [tool.mypy]
 strict = true

src/lep_downloader/config.py

@@ -3,6 +3,8 @@
 
 ARCHIVE_URL = "https://hotenov.com"
 
+JSON_DB_URL = "https://hotenov.com/some_json.json"
+
 LOCAL_ARCHIVE_HTML = "2021-08-10_lep-archive-page-content-pretty.html"
 
 SHORT_LINKS_MAPPING_DICT = {
@@ -23,9 +25,4 @@
 
 EPISODE_LINK_RE = r"https?://((?P<short>wp\.me/p4IuUx-[\w-]+)|(teacherluke\.(co\.uk|wordpress\.com)/(?P<date>\d{4}/\d{2}/\d{2})/))"
 
-LINK_TEXTS_MAPPING = {
-    "https://teacherluke.co.uk/2018/04/18/522-learning-english-at-summer-school-in-the-uk-a-rambling-chat-with-raphael-miller/": "522. Learning English at Summer School in the UK (A Rambling Chat with Raphael Miller)",
-    "https://teacherluke.co.uk/2017/08/14/website-content-lukes-criminal-past-zep-episode-185/": "[Website content] Luke’s Criminal Past (ZEP Episode 185)",
-    "https://teacherluke.co.uk/2017/05/26/i-was-invited-onto-the-english-across-the-pond-podcast/": "[Website content] I was invited onto the “English Across The Pond” Podcast",
-    "https://teacherluke.co.uk/2016/03/20/i-was-invited-onto-craig-wealands-weekly-blab-and-we-talked-about-comedy-video/": "[VIDEO] I was invited onto Craig Wealand’s weekly Blab, and we talked about comedy",
-}
+INVALID_PATH_CHARS_RE = r"[<>:\"/\\\\|?*]"

src/lep_downloader/lep.py

@@ -1 +1,53 @@
 """LEP module for general logic and classes."""
+import json
+import typing as t
+
+
+class LepEpisode(object):
+    """LEP episode class."""
+
+    def __init__(
+        self,
+        episode: int = 0,
+        date: str = "2000-01-01T00:00:00+00:00",
+        url: str = "",
+        post_title: str = "",
+        post_type: str = "",
+        parsing_utc: str = "",
+        index: int = 0,
+        audios: t.Optional[t.List[t.List[str]]] = None,
+        admin_note: str = "",
+    ) -> None:
+        """Default instance of LepEpisode.
+
+        Args:
+            episode (int): Episode number.
+            date (str): Post datetime (default 2000-01-01T00:00:00+00:00).
+            url (str): Final location of post URL.
+            post_title (str): Post title, extracted from tag <a> and safe for windows path.
+            post_type (str): Post type ("AUDIO", "TEXT", etc.).
+            audios (list): List of links lists (for multi-part episodes).
+            parsing_utc (str): Parsing datetime in UTC timezone (with microseconds).
+            index (int): Parsing index: concatenation of URL date and increment (for several posts).
+            admin_note (str): Note for administrator and storing error message (for bad response)
+        """
+        self.episode = episode
+        self.date = date
+        self.url = url
+        self.post_title = post_title
+        self.post_type = post_type
+        self.audios = audios
+        self.parsing_utc = parsing_utc
+        self.index = index
+        self.admin_note = admin_note
+
+
+class LepJsonEncoder(json.JSONEncoder):
+    """Custom JSONEncoder for LepEpisode objects."""
+
+    def default(self, obj: t.Any) -> t.Any:
+        """Override 'default' method for encoding JSON objects."""
+        if isinstance(obj, LepEpisode):
+            return obj.__dict__
+        # Let the base class default method raise the TypeError
+        return json.JSONEncoder.default(self, obj)