fix: nested json table string length

src/gretel_trainer/relational/core.py

@@ -806,13 +806,31 @@ def debug_summary(self) -> dict[str, Any]:
                         "parent_columns": key.parent_columns,
                     }
                 )
-            tables[table] = {
+            table_metadata = {
                 "column_count": len(self.get_table_columns(table)),
                 "primary_key": self.get_primary_key(table),
                 "foreign_key_count": this_table_foreign_key_count,
                 "foreign_keys": foreign_keys,
                 "is_invented_table": self._is_invented(table),
             }
+            if self.is_producer_of_invented_tables(table):
+                table_metadata["invented_table_details"] = {}
+                table_metadata["invented_table_details"]["table_type"] = "producer"
+                producer_metadata = self.get_producer_metadata(table)
+                if producer_metadata is not None:
+                    table_metadata["invented_table_details"][
+                        "json_to_table_mappings"
+                    ] = producer_metadata.table_name_mappings
+            elif self._is_invented(table):
+                table_metadata["invented_table_details"] = {}
+                table_metadata["invented_table_details"]["table_type"] = "invented"
+                invented_table_metadata = self.get_invented_table_metadata(table)
+                if invented_table_metadata is not None:
+                    table_metadata["invented_table_details"][
+                        "json_breadcrumb_path"
+                    ] = invented_table_metadata.json_breadcrumb_path
+            tables[table] = table_metadata
+
         return {
             "foreign_key_count": total_foreign_key_count,
             "max_depth": max_depth,

src/gretel_trainer/relational/json.py

@@ -1,11 +1,11 @@
 from __future__ import annotations
 
-import hashlib
 import logging
 import re
 from dataclasses import dataclass
 from json import JSONDecodeError, dumps, loads
 from typing import Any, Optional, Protocol, Union
+from uuid import uuid4
 
 import numpy as np
 import pandas as pd
@@ -116,15 +116,18 @@ def _is_invented_child_table(table: str, rel_data: _RelationalData) -> bool:
     return imeta is not None and imeta.invented_root_table_name != table
 
 
-def sanitize_str(s):
+def generate_unique_table_name(s: str):
     sanitized_str = "-".join(re.findall(r"[a-zA-Z_0-9]+", s))
-    # Generate suffix from original string, in case of sanitized_str collision
-    unique_suffix = make_suffix(s)
-    return f"{sanitized_str}-{unique_suffix}"
+    # Generate unique suffix to prevent collisions
+    unique_suffix = make_suffix()
+    # Max length for a table/filename is 128 chars
+    return f"{sanitized_str[:80]}_invented_{unique_suffix}"
 
 
-def make_suffix(s):
-    return hashlib.sha256(s.encode("utf-8")).hexdigest()[:10]
+def make_suffix():
+    # Dropping hyphens to save on space since max table/filename length is 128 and we do not need to comply with RFC 4122
+    drop_hyphens = str(uuid4()).split("-")
+    return "".join(drop_hyphens)
 
 
 def jsonencode(df: pd.DataFrame, cols: list[str]) -> pd.DataFrame:
@@ -189,6 +192,7 @@ def get_invented_table_metadata(
 class InventedTableMetadata:
     invented_root_table_name: str
     original_table_name: str
+    json_breadcrumb_path: str
     empty: bool
 
 
@@ -213,7 +217,8 @@ def ingest(
 
     # If we created additional tables (from JSON lists) or added columns (from JSON dicts)
     if len(tables) > 1 or len(tables[0][1].columns) > len(df.columns):
-        mappings = {name: sanitize_str(name) for name, _ in tables}
+        # Map json breadcrumbs to uniquely generated table name
+        mappings = {name: generate_unique_table_name(table_name) for name, _ in tables}
         logger.info(f"Transformed JSON into {len(mappings)} tables for modeling.")
         logger.debug(f"Invented table names: {list(mappings.values())}")
         commands = _generate_commands(
@@ -239,13 +244,13 @@ def _generate_commands(
     Returns lists of keyword arguments designed to be passed to a
     RelationalData instance's _add_single_table and add_foreign_key methods
     """
-    tables_to_add = {table_name_mappings[name]: df for name, df in tables}
     root_table_name = table_name_mappings[original_table_name]
 
     _add_single_table = []
     add_foreign_key = []
 
-    for table_name, table_df in tables_to_add.items():
+    for table_breadcrumb_name, table_df in tables:
+        table_name = table_name_mappings[table_breadcrumb_name]
         if table_name == root_table_name:
             table_pk = original_primary_key + [PRIMARY_KEY_COLUMN]
         else:
@@ -255,6 +260,7 @@ def _generate_commands(
         metadata = InventedTableMetadata(
             invented_root_table_name=root_table_name,
             original_table_name=original_table_name,
+            json_breadcrumb_path=table_breadcrumb_name,
             empty=table_df.empty,
         )
         _add_single_table.append(
@@ -266,7 +272,8 @@ def _generate_commands(
             }
         )
 
-    for table_name, table_df in tables_to_add.items():
+    for table_breadcrumb_name, table_df in tables:
+        table_name = table_name_mappings[table_breadcrumb_name]
         for column in get_id_columns(table_df):
             referred_table = table_name_mappings[
                 get_parent_table_name_from_child_id_column(column)

tests/relational/conftest.py

@@ -20,12 +20,21 @@ def extended_sdk():
 
 
 @pytest.fixture(autouse=True)
-def static_suffix():
+def static_suffix(request):
+    if "no_mock_suffix" in request.keywords:
+        yield
+        return
     with patch("gretel_trainer.relational.json.make_suffix") as make_suffix:
-        make_suffix.return_value = "sfx"
+        # Each call to make_suffix must be unique or there will be table collisions
+        make_suffix.side_effect = [f"sfx{i}" for i in range(1, 50)]
         yield
 
 
+# Doesn't work well as a fixture due to the need for an input param
+def get_invented_table_suffix(make_suffix_execution_number: int):
+    return f"invented_sfx{str(make_suffix_execution_number)}"
+
+
 @pytest.fixture()
 def project():
     with patch(

tests/relational/test_backup.py

@@ -11,6 +11,7 @@
     BackupSyntheticsTrain,
     BackupTransformsTrain,
 )
+from tests.relational.conftest import get_invented_table_suffix
 
 
 def test_backup_relational_data(trips):
@@ -37,32 +38,37 @@ def test_backup_relational_data(trips):
 
 
 def test_backup_relational_data_with_json(documents):
+    purchases_root_invented_table = f"purchases_{get_invented_table_suffix(1)}"
+    purchases_data_years_invented_table = f"purchases_{get_invented_table_suffix(2)}"
+
     expected = BackupRelationalData(
         tables={
             "users": BackupRelationalDataTable(primary_key=["id"]),
             "purchases": BackupRelationalDataTable(
                 primary_key=["id"],
                 producer_metadata={
-                    "invented_root_table_name": "purchases-sfx",
+                    "invented_root_table_name": purchases_root_invented_table,
                     "table_name_mappings": {
-                        "purchases": "purchases-sfx",
-                        "purchases^data>years": "purchases-data-years-sfx",
+                        "purchases": purchases_root_invented_table,
+                        "purchases^data>years": purchases_data_years_invented_table,
                     },
                 },
             ),
-            "purchases-sfx": BackupRelationalDataTable(
+            purchases_root_invented_table: BackupRelationalDataTable(
                 primary_key=["id", "~PRIMARY_KEY_ID~"],
                 invented_table_metadata={
-                    "invented_root_table_name": "purchases-sfx",
+                    "invented_root_table_name": purchases_root_invented_table,
                     "original_table_name": "purchases",
+                    "json_breadcrumb_path": "purchases",
                     "empty": False,
                 },
             ),
-            "purchases-data-years-sfx": BackupRelationalDataTable(
+            purchases_data_years_invented_table: BackupRelationalDataTable(
                 primary_key=["~PRIMARY_KEY_ID~"],
                 invented_table_metadata={
-                    "invented_root_table_name": "purchases-sfx",
+                    "invented_root_table_name": purchases_root_invented_table,
                     "original_table_name": "purchases",
+                    "json_breadcrumb_path": "purchases^data>years",
                     "empty": False,
                 },
             ),
@@ -72,19 +78,19 @@ def test_backup_relational_data_with_json(documents):
             BackupForeignKey(
                 table="payments",
                 constrained_columns=["purchase_id"],
-                referred_table="purchases-sfx",
+                referred_table=purchases_root_invented_table,
                 referred_columns=["id"],
             ),
             BackupForeignKey(
-                table="purchases-sfx",
+                table=purchases_root_invented_table,
                 constrained_columns=["user_id"],
                 referred_table="users",
                 referred_columns=["id"],
             ),
             BackupForeignKey(
-                table="purchases-data-years-sfx",
+                table=purchases_data_years_invented_table,
                 constrained_columns=["purchases~id"],
-                referred_table="purchases-sfx",
+                referred_table=purchases_root_invented_table,
                 referred_columns=["~PRIMARY_KEY_ID~"],
             ),
         ],