fix: nested json table string length

pyproject.toml

@@ -0,0 +1,4 @@
+[tool.pytest.ini_options]
+markers = [
+    "no_mock_suffix",
+]

src/gretel_trainer/relational/core.py

@@ -806,13 +806,27 @@ def debug_summary(self) -> dict[str, Any]:
                         "parent_columns": key.parent_columns,
                     }
                 )
-            tables[table] = {
+            table_metadata = {
                 "column_count": len(self.get_table_columns(table)),
                 "primary_key": self.get_primary_key(table),
                 "foreign_key_count": this_table_foreign_key_count,
                 "foreign_keys": foreign_keys,
                 "is_invented_table": self._is_invented(table),
             }
+            if (producer_metadata := self.get_producer_metadata(table)) is not None:
+                table_metadata["invented_table_details"] = {
+                    "table_type": "producer",
+                    "json_to_table_mappings": producer_metadata.table_name_mappings,
+                }
+            elif (
+                invented_table_metadata := self.get_invented_table_metadata(table)
+            ) is not None:
+                table_metadata["invented_table_details"] = {
+                    "table_type": "invented",
+                    "json_breadcrumb_path": invented_table_metadata.json_breadcrumb_path,
+                }
+            tables[table] = table_metadata
+
         return {
             "foreign_key_count": total_foreign_key_count,
             "max_depth": max_depth,

src/gretel_trainer/relational/json.py

@@ -1,11 +1,11 @@
 from __future__ import annotations
 
-import hashlib
 import logging
 import re
 from dataclasses import dataclass
 from json import JSONDecodeError, dumps, loads
 from typing import Any, Optional, Protocol, Union
+from uuid import uuid4
 
 import numpy as np
 import pandas as pd
@@ -116,15 +116,16 @@ def _is_invented_child_table(table: str, rel_data: _RelationalData) -> bool:
     return imeta is not None and imeta.invented_root_table_name != table
 
 
-def sanitize_str(s):
+def generate_unique_table_name(s: str):
     sanitized_str = "-".join(re.findall(r"[a-zA-Z_0-9]+", s))
-    # Generate suffix from original string, in case of sanitized_str collision
-    unique_suffix = make_suffix(s)
-    return f"{sanitized_str}-{unique_suffix}"
+    # Generate unique suffix to prevent collisions
+    unique_suffix = make_suffix()
+    # Max length for a table/filename is 128 chars
+    return f"{sanitized_str[:80]}_invented_{unique_suffix}"
 
 
-def make_suffix(s):
-    return hashlib.sha256(s.encode("utf-8")).hexdigest()[:10]
+def make_suffix():
+    return uuid4().hex
 
 
 def jsonencode(df: pd.DataFrame, cols: list[str]) -> pd.DataFrame:
@@ -189,6 +190,7 @@ def get_invented_table_metadata(
 class InventedTableMetadata:
     invented_root_table_name: str
     original_table_name: str
+    json_breadcrumb_path: str
     empty: bool
 
 
@@ -213,7 +215,8 @@ def ingest(
 
     # If we created additional tables (from JSON lists) or added columns (from JSON dicts)
     if len(tables) > 1 or len(tables[0][1].columns) > len(df.columns):
-        mappings = {name: sanitize_str(name) for name, _ in tables}
+        # Map json breadcrumbs to uniquely generated table name
+        mappings = {name: generate_unique_table_name(table_name) for name, _ in tables}
         logger.info(f"Transformed JSON into {len(mappings)} tables for modeling.")
         logger.debug(f"Invented table names: {list(mappings.values())}")
         commands = _generate_commands(
@@ -239,13 +242,13 @@ def _generate_commands(
     Returns lists of keyword arguments designed to be passed to a
     RelationalData instance's _add_single_table and add_foreign_key methods
     """
-    tables_to_add = {table_name_mappings[name]: df for name, df in tables}
     root_table_name = table_name_mappings[original_table_name]
 
     _add_single_table = []
     add_foreign_key = []
 
-    for table_name, table_df in tables_to_add.items():
+    for table_breadcrumb_name, table_df in tables:
+        table_name = table_name_mappings[table_breadcrumb_name]
         if table_name == root_table_name:
             table_pk = original_primary_key + [PRIMARY_KEY_COLUMN]
         else:
@@ -255,6 +258,7 @@ def _generate_commands(
         metadata = InventedTableMetadata(
             invented_root_table_name=root_table_name,
             original_table_name=original_table_name,
+            json_breadcrumb_path=table_breadcrumb_name,
             empty=table_df.empty,
         )
         _add_single_table.append(
@@ -266,7 +270,8 @@ def _generate_commands(
             }
         )
 
-    for table_name, table_df in tables_to_add.items():
+    for table_breadcrumb_name, table_df in tables:
+        table_name = table_name_mappings[table_breadcrumb_name]
         for column in get_id_columns(table_df):
             referred_table = table_name_mappings[
                 get_parent_table_name_from_child_id_column(column)

tests/relational/conftest.py

@@ -1,6 +1,8 @@
+import itertools
 import sqlite3
 import tempfile
 from pathlib import Path
+from typing import Generator
 from unittest.mock import Mock, patch
 
 import pandas as pd
@@ -20,10 +22,19 @@ def extended_sdk():
 
 
 @pytest.fixture(autouse=True)
-def static_suffix():
-    with patch("gretel_trainer.relational.json.make_suffix") as make_suffix:
-        make_suffix.return_value = "sfx"
+def static_suffix(request):
+    if "no_mock_suffix" in request.keywords:
         yield
+        return
+    with patch("gretel_trainer.relational.json.make_suffix") as make_suffix:
+        # Each call to make_suffix must be unique or there will be table collisions
+        make_suffix.side_effect = itertools.count(start=1)
+        yield make_suffix
+
+
+# Doesn't work well as a fixture due to the need for an input param
+def get_invented_table_suffix(make_suffix_execution_number: int):
+    return f"invented_{str(make_suffix_execution_number)}"
 
 
 @pytest.fixture()
@@ -71,37 +82,37 @@ def tmpfile():
 
 
 @pytest.fixture()
-def pets(tmpdir):
+def pets(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("pets").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def ecom(tmpdir):
+def ecom(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("ecom").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def mutagenesis(tmpdir):
+def mutagenesis(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("mutagenesis").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def tpch(tmpdir):
+def tpch(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("tpch").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def art(tmpdir):
+def art(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("art").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def documents(tmpdir):
+def documents(tmpdir) -> Generator[RelationalData, None, None]:
     yield _rel_data_connector("documents").extract(storage_dir=tmpdir)
 
 
 @pytest.fixture()
-def trips(tmpdir):
+def trips(tmpdir) -> Generator[RelationalData, None, None]:
     with tempfile.NamedTemporaryFile() as tmpfile:
         data = pd.DataFrame(
             data={

tests/relational/test_backup.py

@@ -11,6 +11,7 @@
     BackupSyntheticsTrain,
     BackupTransformsTrain,
 )
+from tests.relational.conftest import get_invented_table_suffix
 
 
 def test_backup_relational_data(trips):
@@ -37,32 +38,37 @@ def test_backup_relational_data(trips):
 
 
 def test_backup_relational_data_with_json(documents):
+    purchases_root_invented_table = f"purchases_{get_invented_table_suffix(1)}"
+    purchases_data_years_invented_table = f"purchases_{get_invented_table_suffix(2)}"
+
     expected = BackupRelationalData(
         tables={
             "users": BackupRelationalDataTable(primary_key=["id"]),
             "purchases": BackupRelationalDataTable(
                 primary_key=["id"],
                 producer_metadata={
-                    "invented_root_table_name": "purchases-sfx",
+                    "invented_root_table_name": purchases_root_invented_table,
                     "table_name_mappings": {
-                        "purchases": "purchases-sfx",
-                        "purchases^data>years": "purchases-data-years-sfx",
+                        "purchases": purchases_root_invented_table,
+                        "purchases^data>years": purchases_data_years_invented_table,
                     },
                 },
             ),
-            "purchases-sfx": BackupRelationalDataTable(
+            purchases_root_invented_table: BackupRelationalDataTable(
                 primary_key=["id", "~PRIMARY_KEY_ID~"],
                 invented_table_metadata={
-                    "invented_root_table_name": "purchases-sfx",
+                    "invented_root_table_name": purchases_root_invented_table,
                     "original_table_name": "purchases",
+                    "json_breadcrumb_path": "purchases",
                     "empty": False,
                 },
             ),
-            "purchases-data-years-sfx": BackupRelationalDataTable(
+            purchases_data_years_invented_table: BackupRelationalDataTable(
                 primary_key=["~PRIMARY_KEY_ID~"],
                 invented_table_metadata={
-                    "invented_root_table_name": "purchases-sfx",
+                    "invented_root_table_name": purchases_root_invented_table,
                     "original_table_name": "purchases",
+                    "json_breadcrumb_path": "purchases^data>years",
                     "empty": False,
                 },
             ),
@@ -72,19 +78,19 @@ def test_backup_relational_data_with_json(documents):
             BackupForeignKey(
                 table="payments",
                 constrained_columns=["purchase_id"],
-                referred_table="purchases-sfx",
+                referred_table=purchases_root_invented_table,
                 referred_columns=["id"],
             ),
             BackupForeignKey(
-                table="purchases-sfx",
+                table=purchases_root_invented_table,
                 constrained_columns=["user_id"],
                 referred_table="users",
                 referred_columns=["id"],
             ),
             BackupForeignKey(
-                table="purchases-data-years-sfx",
+                table=purchases_data_years_invented_table,
                 constrained_columns=["purchases~id"],
-                referred_table="purchases-sfx",
+                referred_table=purchases_root_invented_table,
                 referred_columns=["~PRIMARY_KEY_ID~"],
             ),
         ],