Add pytest, static test file

data_ingest/.gitattributes

@@ -0,0 +1 @@
+tests/scrape_responses/*.html linguist-generated

data_ingest/.gitignore

@@ -1 +1,2 @@
 venv/*
+__pycache__/

data_ingest/.python-version

@@ -0,0 +1 @@
+3.12

data_ingest/main.py

@@ -0,0 +1,2 @@
+if __name__ == "__main__":
+    print("starting import")

data_ingest/pyproject.toml

@@ -0,0 +1,13 @@
+[project]
+name = "data-ingest"
+version = "0.1.0"
+description = "Scripts to scrape data and insert into database"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "imslp>=0.2.3",
+    "pytest>=8.3.3",
+    "ruff>=0.7.4",
+]
+[tool.pytest.ini_options]
+pythonpath = ["."]

data_ingest/tests/test_parse_movements.py

@@ -0,0 +1,15 @@
+from bs4 import BeautifulSoup
+
+from utils import parse_movements
+
+
+def test_parse_movements():
+    with open("tests/scrape_responses/pieces.html", "r") as file:
+        html_content = file.read()
+        soup = BeautifulSoup(html_content, "html.parser")
+        general_info_div = soup.find("div", class_="wi_body")
+        if general_info_div is None:
+            raise ValueError("Could not find 'div' with class 'wi_body'")
+        result = parse_movements(general_info_div)
+        assert isinstance( result, list)
+        assert set(result[0].keys()) == ['name', 'number']

data_ingest/utils.py

@@ -0,0 +1,64 @@
+import re
+
+from bs4 import NavigableString, Tag
+
+
+def parse_movements(data: Tag | NavigableString):
+    movements = []
+
+    print(f"data: {data}")
+    movement_list = data.find("ol")
+    if movement_list:
+        print("movement list")
+        for index, li in enumerate(movement_list.find_all("li")):
+            line = li.get_text(strip=True).replace("\xa0", " ")
+            number = 3
+
+            if "(" in line and line.endswith(")"):
+                name, key_signature = line.rsplit("(", 1)
+                key_signature = key_signature.rstrip(")").replace("\xa0", " ")
+                movements.append(
+                    {
+                        "type": "movement",
+                        "number": number,
+                        "name": name.strip(),
+                        "key_signature": key_signature.strip(),
+                    }
+                )
+            else:
+                movements.append(
+                    {
+                        "type": "movement",
+                        "number": number,
+                        "name": line.strip(),
+                        "key_signature": None,
+                    }
+                )
+    piece_list = data.find("dl")
+    if piece_list:
+        for index, dd in enumerate(piece_list.find_all("dd")):
+            line = dd.get_text(strip=True).replace("\xa0", " ")
+            number = 3
+
+            if "(" in line and line.endswith(")"):
+                name, key_signature = line.rsplit("(", 1)
+                key_signature = key_signature.rstrip(")").replace("\xa0", " ")
+                movements.append(
+                    {
+                        "type": "piece",
+                        "number": number,
+                        "name": name.strip(),
+                        "key_signature": key_signature.strip(),
+                    }
+                )
+            else:
+                movements.append(
+                    {
+                        "type": "piece",
+                        "number": number,
+                        "name": line.strip(),
+                        "key_signature": None,
+                    }
+                )
+    print(f"movements {movements}")
+    return movements