fix: nested json table string length (#135)

* fix: nested json table string length

In cases where we are parsing nested json with lengthy key strings the
combination of key strings in the breadcrumb based table name string
would result in an error when trying to train the model due to a maximum
json filename length of 128 chars. Here we utilize the already existing
make_suffix function (which ensures no sanitized_str collisions) and we
truncate the length of the table name to ensure we do not exceed the 128
char limit.

* feat: remove json breadcrumbs from table names

Instead of sanitizing the full json path to an invented table, we create
a table name based on the producer table with a uuid to prevent
collisions, and we map the full json path to the unique table string.

* feat: store json table mapping in debug_summary

When nested json structures are extracted to individual dataframes they
will be written to resulting individual tables. Since we are creating
table names with the convention <producer_table>_invented_<uuid> we now
store the mapping details for tables in the producer table's debug
summary, as well as the json breadcrumb path in each invented table's
individual debug summary.

pyproject.toml

@@ -0,0 +1,4 @@
+[tool.pytest.ini_options]
+markers = [
+    "no_mock_suffix",
+]

src/gretel_trainer/relational/core.py

@@ -806,13 +806,27 @@ def debug_summary(self) -> dict[str, Any]:
                         "parent_columns": key.parent_columns,
                     }
                 )
-            tables[table] = {
+            table_metadata = {
                 "column_count": len(self.get_table_columns(table)),
                 "primary_key": self.get_primary_key(table),
                 "foreign_key_count": this_table_foreign_key_count,
                 "foreign_keys": foreign_keys,
                 "is_invented_table": self._is_invented(table),
             }
+            if (producer_metadata := self.get_producer_metadata(table)) is not None:
+                table_metadata["invented_table_details"] = {
+                    "table_type": "producer",
+                    "json_to_table_mappings": producer_metadata.table_name_mappings,
+                }
+            elif (
+                invented_table_metadata := self.get_invented_table_metadata(table)
+            ) is not None:
+                table_metadata["invented_table_details"] = {
+                    "table_type": "invented",
+                    "json_breadcrumb_path": invented_table_metadata.json_breadcrumb_path,
+                }
+            tables[table] = table_metadata
+
         return {
             "foreign_key_count": total_foreign_key_count,
             "max_depth": max_depth,

src/gretel_trainer/relational/json.py

@@ -1,11 +1,11 @@
 from __future__ import annotations
 
-import hashlib
 import logging
 import re
 from dataclasses import dataclass
 from json import JSONDecodeError, dumps, loads
 from typing import Any, Optional, Protocol, Union
+from uuid import uuid4
 
 import numpy as np
 import pandas as pd
@@ -116,15 +116,16 @@ def _is_invented_child_table(table: str, rel_data: _RelationalData) -> bool:
     return imeta is not None and imeta.invented_root_table_name != table
 
 
-def sanitize_str(s):
+def generate_unique_table_name(s: str):
     sanitized_str = "-".join(re.findall(r"[a-zA-Z_0-9]+", s))
-    # Generate suffix from original string, in case of sanitized_str collision
-    unique_suffix = make_suffix(s)
-    return f"{sanitized_str}-{unique_suffix}"
+    # Generate unique suffix to prevent collisions
+    unique_suffix = make_suffix()
+    # Max length for a table/filename is 128 chars
+    return f"{sanitized_str[:80]}_invented_{unique_suffix}"
 
 
-def make_suffix(s):
-    return hashlib.sha256(s.encode("utf-8")).hexdigest()[:10]
+def make_suffix():
+    return uuid4().hex
 
 
 def jsonencode(df: pd.DataFrame, cols: list[str]) -> pd.DataFrame:
@@ -189,6 +190,7 @@ def get_invented_table_metadata(
 class InventedTableMetadata:
     invented_root_table_name: str
     original_table_name: str
+    json_breadcrumb_path: str
     empty: bool
 
 
@@ -213,7 +215,8 @@ def ingest(
 
     # If we created additional tables (from JSON lists) or added columns (from JSON dicts)
     if len(tables) > 1 or len(tables[0][1].columns) > len(df.columns):
-        mappings = {name: sanitize_str(name) for name, _ in tables}
+        # Map json breadcrumbs to uniquely generated table name
+        mappings = {name: generate_unique_table_name(table_name) for name, _ in tables}
         logger.info(f"Transformed JSON into {len(mappings)} tables for modeling.")
         logger.debug(f"Invented table names: {list(mappings.values())}")
         commands = _generate_commands(
@@ -239,13 +242,13 @@ def _generate_commands(
     Returns lists of keyword arguments designed to be passed to a
     RelationalData instance's _add_single_table and add_foreign_key methods
     """
-    tables_to_add = {table_name_mappings[name]: df for name, df in tables}
     root_table_name = table_name_mappings[original_table_name]
 
     _add_single_table = []
     add_foreign_key = []
 
-    for table_name, table_df in tables_to_add.items():
+    for table_breadcrumb_name, table_df in tables:
+        table_name = table_name_mappings[table_breadcrumb_name]
         if table_name == root_table_name:
             table_pk = original_primary_key + [PRIMARY_KEY_COLUMN]
         else:
@@ -255,6 +258,7 @@ def _generate_commands(
         metadata = InventedTableMetadata(
             invented_root_table_name=root_table_name,
             original_table_name=original_table_name,
+            json_breadcrumb_path=table_breadcrumb_name,
             empty=table_df.empty,
         )
         _add_single_table.append(
@@ -266,7 +270,8 @@ def _generate_commands(
             }
         )
 
-    for table_name, table_df in tables_to_add.items():
+    for table_breadcrumb_name, table_df in tables:
+        table_name = table_name_mappings[table_breadcrumb_name]
         for column in get_id_columns(table_df):
             referred_table = table_name_mappings[
                 get_parent_table_name_from_child_id_column(column)

tests/relational/conftest.py

@@ -1,6 +1,8 @@
+import itertools
 import sqlite3
 import tempfile
 from pathlib import Path
+from typing import Generator
 from unittest.mock import Mock, patch
 
 import pandas as pd
@@ -20,10 +22,19 @@ def extended_sdk():
 
 
 @pytest.fixture(autouse=True)
-def static_suffix():
-    with patch("gretel_trainer.relational.json.make_suffix") as make_suffix:
-        make_suffix.return_value = "sfx"
+def static_suffix(request):
+    if "no_mock_suffix" in request.keywords:
         yield
+        return
+    with patch("gretel_trainer.relational.json.make_suffix") as make_suffix:
+        # Each call to make_suffix must be unique or there will be table collisions
+        make_suffix.side_effect = itertools.count(start=1)
+        yield make_suffix
+
+
+# Doesn't work well as a fixture due to the need for an input param
+def get_invented_table_suffix(make_suffix_execution_number: int):
+    return f"invented_{str(make_suffix_execution_number)}"
 
 
 @pytest.fixture()
@@ -71,37 +82,37 @@ def tmpfile():
 
 
 @pytest.fixture()
-def pets(tmpdir):
+def pets(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("pets").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def ecom(tmpdir):
+def ecom(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("ecom").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def mutagenesis(tmpdir):
+def mutagenesis(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("mutagenesis").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def tpch(tmpdir):
+def tpch(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("tpch").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def art(tmpdir):
+def art(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("art").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def documents(tmpdir):
+def documents(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("documents").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def trips(tmpdir):
+def trips(tmpdir) -> Generator[RelationalData, None, None]:
     with tempfile.NamedTemporaryFile() as tmpfile:
         data = pd.DataFrame(
             data={

tests/relational/test_backup.py

@@ -11,6 +11,7 @@
     BackupSyntheticsTrain,
     BackupTransformsTrain,
 )
+from tests.relational.conftest import get_invented_table_suffix
 
 
 def test_backup_relational_data(trips):
@@ -37,32 +38,37 @@ def test_backup_relational_data(trips):
 
 
 def test_backup_relational_data_with_json(documents):
+    purchases_root_invented_table = f"purchases_{get_invented_table_suffix(1)}"
+    purchases_data_years_invented_table = f"purchases_{get_invented_table_suffix(2)}"
+
     expected = BackupRelationalData(
         tables={
             "users": BackupRelationalDataTable(primary_key=["id"]),
             "purchases": BackupRelationalDataTable(
                 primary_key=["id"],
                 producer_metadata={
-                    "invented_root_table_name": "purchases-sfx",
+                    "invented_root_table_name": purchases_root_invented_table,
                     "table_name_mappings": {
-                        "purchases": "purchases-sfx",
-                        "purchases^data>years": "purchases-data-years-sfx",
+                        "purchases": purchases_root_invented_table,
+                        "purchases^data>years": purchases_data_years_invented_table,
                     },
                 },
             ),
-            "purchases-sfx": BackupRelationalDataTable(
+            purchases_root_invented_table: BackupRelationalDataTable(
                 primary_key=["id", "~PRIMARY_KEY_ID~"],
                 invented_table_metadata={
-                    "invented_root_table_name": "purchases-sfx",
+                    "invented_root_table_name": purchases_root_invented_table,
                     "original_table_name": "purchases",
+                    "json_breadcrumb_path": "purchases",
                     "empty": False,
                 },
             ),
-            "purchases-data-years-sfx": BackupRelationalDataTable(
+            purchases_data_years_invented_table: BackupRelationalDataTable(
                 primary_key=["~PRIMARY_KEY_ID~"],
                 invented_table_metadata={
-                    "invented_root_table_name": "purchases-sfx",
+                    "invented_root_table_name": purchases_root_invented_table,
                     "original_table_name": "purchases",
+                    "json_breadcrumb_path": "purchases^data>years",
                     "empty": False,
                 },
             ),
@@ -72,19 +78,19 @@ def test_backup_relational_data_with_json(documents):
             BackupForeignKey(
                 table="payments",
                 constrained_columns=["purchase_id"],
-                referred_table="purchases-sfx",
+                referred_table=purchases_root_invented_table,
                 referred_columns=["id"],
             ),
             BackupForeignKey(
-                table="purchases-sfx",
+                table=purchases_root_invented_table,
                 constrained_columns=["user_id"],
                 referred_table="users",
                 referred_columns=["id"],
             ),
             BackupForeignKey(
-                table="purchases-data-years-sfx",
+                table=purchases_data_years_invented_table,
                 constrained_columns=["purchases~id"],
-                referred_table="purchases-sfx",
+                referred_table=purchases_root_invented_table,
                 referred_columns=["~PRIMARY_KEY_ID~"],
             ),
         ],