From c3add0fd7b8e3883acfa65b75562a67b9bde934d Mon Sep 17 00:00:00 2001 From: Bartlomiej Rutkowski Date: Tue, 27 Aug 2024 12:48:11 +0200 Subject: [PATCH 1/3] Add failing test --- .gitignore | 2 +- .idea/.gitignore | 5 ++ .idea/misc.xml | 7 ++ .idea/modules.xml | 8 +++ .idea/pgsync.iml | 12 ++++ .idea/vcs.xml | 6 ++ tests/conftest.py | 23 ++++++ tests/test_sync_no_fk.py | 147 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/pgsync.iml create mode 100644 .idea/vcs.xml create mode 100644 tests/test_sync_no_fk.py diff --git a/.gitignore b/.gitignore index 500ba62d..8a6f1fe7 100644 --- a/.gitignore +++ b/.gitignore @@ -118,4 +118,4 @@ old examples/tiers/ # vscode -.vscode/ \ No newline at end of file +.vscode/ diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..b58b603f --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..b98c4083 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..65ef583a --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/pgsync.iml b/.idea/pgsync.iml new file mode 100644 index 00000000..24643cc3 --- /dev/null +++ b/.idea/pgsync.iml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 30a2be4d..10da6620 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ import pytest import sqlalchemy as sa +import sqlalchemy.schema from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, sessionmaker from sqlalchemy.schema import UniqueConstraint @@ -398,6 +399,15 @@ class Rating(base): return Rating +@pytest.fixture(scope="session") +def review_cls(base, book_cls): + class Review(base): + __tablename__ = "review" + id: Mapped[int] = mapped_column(sa.Integer, primary_key=True) + book_isbn: Mapped[str] = mapped_column(sa.String, nullable=False) + text: Mapped[str] = mapped_column(sa.String, nullable=False) + + return Review @pytest.fixture(scope="session") def group_cls(base): @@ -453,6 +463,7 @@ def model_mapping( contact_item_cls, book_group_cls, group_cls, + review_cls, ): return { "cities": city_cls, @@ -474,11 +485,16 @@ def model_mapping( "contact_items": contact_item_cls, "book_groups": book_group_cls, "groups": group_cls, + "reviews": review_cls, } @pytest.fixture(scope="session") def table_creator(base, connection, model_mapping): + with connection.engine.connect() as conn: + conn.execute(sqlalchemy.schema.CreateSchema(name="review", if_not_exists=True)) + conn.commit() + sa.orm.configure_mappers() with connection.engine.connect() as conn: base.metadata.create_all(connection.engine) @@ -520,6 +536,7 @@ def dataset( rating_cls, book_group_cls, group_cls, + review_cls, ): eu_continent = continent_cls(name="Europe") na_continent = continent_cls(name="North America") @@ -756,4 +773,10 @@ def dataset( ] session.add_all(ratings) + reviews = [ + review_cls(book_isbn="001", text="Great book"), + ] + + session.add_all(reviews) + session.commit() diff --git a/tests/test_sync_no_fk.py b/tests/test_sync_no_fk.py new file mode 100644 index 00000000..73ddb9f5 --- /dev/null +++ b/tests/test_sync_no_fk.py @@ -0,0 +1,147 @@ +import pytest +import psycopg2 +import typing as t + +from pgsync.base import subtransactions, Payload +from pgsync.singleton import Singleton +from pgsync.node import Tree + +@pytest.mark.usefixtures("table_creator") +class TestSyncBetweenSchemas(object): + @pytest.fixture(scope="function") + def data(self, sync, book_cls, review_cls): + session = sync.session + + books = [ + book_cls( + isbn="abc", + title="The Tiger Club", + description="Tigers are fierce creatures", + ) + ] + + with subtransactions(session): + session.add_all(books) + + reviews = [ + review_cls( + text="Great book", + book_isbn="abc", + ), + ] + + with subtransactions(session): + session.add_all(reviews) + + sync.logical_slot_get_changes( + f"{sync.database}_testdb", + upto_nchanges=None, + ) + Singleton._instances = {} + + yield ( + books, + reviews, + ) + + with subtransactions(session): + conn = session.connection().engine.connect().connection + conn.set_isolation_level( + psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT + ) + cursor = conn.cursor() + channel = session.connection().engine.url.database + cursor.execute(f"UNLISTEN {channel}") + + with subtransactions(session): + sync.truncate_tables( + [book_cls.__table__.name, review_cls.__table__.name], + ) + + sync.logical_slot_get_changes( + f"{sync.database}_testdb", + upto_nchanges=None, + ) + + + sync.redis.delete() + session.connection().engine.connect().close() + session.connection().engine.dispose() + sync.search_client.close() + + def test_sync_between_schemas(self, sync, data, review_cls): + nodes = { + "label": "books", + "table": "book", + "columns": ["isbn", "title", "description"], + "children": [ + { + "label": "reviews", + "table": "review", + "columns": ["id", "text"], + "relationship": { + "variant": "object", + "type": "one_to_many", + "foreign_key": { + "child": [ + "book_isbn" + ], + "parent": [ + "isbn" + ] + } + } + } + ] + } + sync.tree = Tree(sync.models, nodes) + docs = list(sync.sync()) + assert docs[0]["_id"] == "abc" + assert docs[0]["_source"] == { + "_meta": {"review": {"id": [1]}}, + "isbn": "abc", + "title": "The Tiger Club", + "description": "Tigers are fierce creatures", + "reviews": [{ + "id": 1, + "text": "Great book", + }] + } + sync.search_client.bulk( + sync.index, + docs, + ) + + session = sync.session + review = review_cls( + text="The best book", + book_isbn="abc", + ) + + with subtransactions(session): + # Insert a new review + session.add(review) + session.commit() + + payloads: t.List[Payload] = [ + Payload( + tg_op="INSERT", + table="review", + schema="public", + new={"id": "2", "book_isbn": "abc", "text": "The best book"}, + xmin=1234, + ) + ] + sync.on_publish(payloads) + sync.search_client.refresh("testdb") + docs = sync.search_client.search( + "testdb", body={"query": {"match_all": {}}} + ) + assert docs["hits"]["total"]["value"] == 1 + assert docs["hits"]["hits"][0]["_source"]["reviews"] == [ + {"id": 1, "text": "Great book"}, + {"id": 2, "text": "The best book"}, + ] + + + From bb21737520754144d5e55381756b86c919316055 Mon Sep 17 00:00:00 2001 From: Bartlomiej Rutkowski Date: Tue, 27 Aug 2024 12:49:02 +0200 Subject: [PATCH 2/3] Remove .idea --- .gitignore | 3 +++ .idea/.gitignore | 5 ----- .idea/misc.xml | 7 ------- .idea/modules.xml | 8 -------- .idea/pgsync.iml | 12 ------------ .idea/vcs.xml | 6 ------ 6 files changed, 3 insertions(+), 38 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/pgsync.iml delete mode 100644 .idea/vcs.xml diff --git a/.gitignore b/.gitignore index 8a6f1fe7..75535b2d 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,6 @@ examples/tiers/ # vscode .vscode/ + +# Jetbrains +.idea/ diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index b58b603f..00000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index b98c4083..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 65ef583a..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/pgsync.iml b/.idea/pgsync.iml deleted file mode 100644 index 24643cc3..00000000 --- a/.idea/pgsync.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddf..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From f64641527af54846e2186f469faa661d12ce6904 Mon Sep 17 00:00:00 2001 From: Bartlomiej Rutkowski Date: Tue, 27 Aug 2024 13:07:43 +0200 Subject: [PATCH 3/3] Fix test --- tests/conftest.py | 5 ----- tests/test_sync_no_fk.py | 14 ++++---------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 10da6620..e80b3b96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,6 @@ import pytest import sqlalchemy as sa -import sqlalchemy.schema from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, sessionmaker from sqlalchemy.schema import UniqueConstraint @@ -491,10 +490,6 @@ def model_mapping( @pytest.fixture(scope="session") def table_creator(base, connection, model_mapping): - with connection.engine.connect() as conn: - conn.execute(sqlalchemy.schema.CreateSchema(name="review", if_not_exists=True)) - conn.commit() - sa.orm.configure_mappers() with connection.engine.connect() as conn: base.metadata.create_all(connection.engine) diff --git a/tests/test_sync_no_fk.py b/tests/test_sync_no_fk.py index 73ddb9f5..745bc790 100644 --- a/tests/test_sync_no_fk.py +++ b/tests/test_sync_no_fk.py @@ -97,16 +97,10 @@ def test_sync_between_schemas(self, sync, data, review_cls): sync.tree = Tree(sync.models, nodes) docs = list(sync.sync()) assert docs[0]["_id"] == "abc" - assert docs[0]["_source"] == { - "_meta": {"review": {"id": [1]}}, - "isbn": "abc", - "title": "The Tiger Club", - "description": "Tigers are fierce creatures", - "reviews": [{ - "id": 1, - "text": "Great book", - }] - } + assert docs[0]["_source"]["reviews"] == [{ + "id": 1, + "text": "Great book", + }] sync.search_client.bulk( sync.index, docs,