From 101b229321d4eed2c27cac21a5669b55ed505b62 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 7 Sep 2023 15:26:52 -0400 Subject: [PATCH 01/38] initial working test --- pandas/tests/io/test_sql.py | 537 +++++++++++++++++------------------- 1 file changed, 259 insertions(+), 278 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index bbdb22955297e..75e1763cb1f51 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -397,6 +397,56 @@ def test_frame3(): return DataFrame(data, columns=columns) +def get_all_views(conn): + if isinstance(conn, sqlite3.Connection): + c = conn.execute("SELECT name FROM sqlite_master WHERE type='view'") + return [view[0] for view in c.fetchall()] + else: + from sqlalchemy import inspect + + return inspect(conn).get_view_names() + + +def get_all_tables(conn): + if isinstance(conn, sqlite3.Connection): + c = conn.execute("SELECT name FROM sqlite_master WHERE type='table'") + return [table[0] for table in c.fetchall()] + else: + from sqlalchemy import inspect + + return inspect(conn).get_table_names() + + +def drop_table( + table_name: str, + conn: sqlite3.Connection | sqlalchemy.engine.Engine | sqlalchemy.engine.Connection, +): + if isinstance(conn, sqlite3.Connection): + conn.execute(f"DROP TABLE IF EXISTS {sql._get_valid_sqlite_name(table_name)}") + conn.commit() + else: + sql.SQLDatabase(conn).drop_table(table_name) + + +def drop_view( + view_name: str, + conn: sqlite3.Connection | sqlalchemy.engine.Engine | sqlalchemy.engine.Connection, +): + if isinstance(conn, sqlite3.Connection): + conn.execute(f"DROP VIEW IF EXISTS {sql._get_valid_sqlite_name(view_name)}") + conn.commit() + else: + quoted_view = conn.engine.dialect.identifier_preparer.quote_identifier( + view_name + ) + stmt = sqlalchemy.text(f"DROP VIEW IF EXISTS {quoted_view}") + if isinstance(conn, sqlalchemy.engine.Engine): + with conn.connect() as conn: + conn.execute(stmt) + else: + conn.execute(stmt) + + @pytest.fixture def mysql_pymysql_engine(iris_path, types_data): sqlalchemy = pytest.importorskip("sqlalchemy") @@ -418,8 +468,10 @@ def mysql_pymysql_engine(iris_path, types_data): yield engine with engine.connect() as conn: with conn.begin(): - stmt = sqlalchemy.text("DROP TABLE IF EXISTS test_frame;") - conn.execute(stmt) + for view in get_all_views(conn): + drop_view(view, conn) + for tbl in get_all_tables(conn): + drop_table(tbl, conn) engine.dispose() @@ -447,8 +499,10 @@ def postgresql_psycopg2_engine(iris_path, types_data): yield engine with engine.connect() as conn: with conn.begin(): - stmt = sqlalchemy.text("DROP TABLE IF EXISTS test_frame;") - conn.execute(stmt) + for view in get_all_views(conn): + drop_view(view, conn) + for tbl in get_all_tables(conn): + drop_table(tbl, conn) engine.dispose() @@ -481,6 +535,10 @@ def sqlite_engine(sqlite_str, iris_path, types_data): create_and_load_types(engine, types_data, "sqlite") yield engine + for view in get_all_views(engine): + drop_view(view, engine) + for tbl in get_all_tables(engine): + drop_table(tbl, engine) engine.dispose() @@ -1041,79 +1099,6 @@ def test_execute_deprecated(sqlite_buildin_iris): sql.execute("select * from iris", sqlite_buildin_iris) -class MixInBase: - def teardown_method(self): - # if setup fails, there may not be a connection to close. - if hasattr(self, "conn"): - self.conn.close() - # use a fresh connection to ensure we can drop all tables. - try: - conn = self.connect() - except (sqlalchemy.exc.OperationalError, sqlite3.OperationalError): - pass - else: - with conn: - for view in self._get_all_views(conn): - self.drop_view(view, conn) - for tbl in self._get_all_tables(conn): - self.drop_table(tbl, conn) - - -class SQLiteMixIn(MixInBase): - def connect(self): - return sqlite3.connect(":memory:") - - def drop_table(self, table_name, conn): - conn.execute(f"DROP TABLE IF EXISTS {sql._get_valid_sqlite_name(table_name)}") - conn.commit() - - def _get_all_tables(self, conn): - c = conn.execute("SELECT name FROM sqlite_master WHERE type='table'") - return [table[0] for table in c.fetchall()] - - def drop_view(self, view_name, conn): - conn.execute(f"DROP VIEW IF EXISTS {sql._get_valid_sqlite_name(view_name)}") - conn.commit() - - def _get_all_views(self, conn): - c = conn.execute("SELECT name FROM sqlite_master WHERE type='view'") - return [view[0] for view in c.fetchall()] - - -class SQLAlchemyMixIn(MixInBase): - @classmethod - def teardown_class(cls): - cls.engine.dispose() - - def connect(self): - return self.engine.connect() - - def drop_table(self, table_name, conn): - if conn.in_transaction(): - conn.get_transaction().rollback() - with conn.begin(): - sql.SQLDatabase(conn).drop_table(table_name) - - def _get_all_tables(self, conn): - from sqlalchemy import inspect - - return inspect(conn).get_table_names() - - def drop_view(self, view_name, conn): - quoted_view = conn.engine.dialect.identifier_preparer.quote_identifier( - view_name - ) - if conn.in_transaction(): - conn.get_transaction().rollback() - with conn.begin(): - conn.exec_driver_sql(f"DROP VIEW IF EXISTS {quoted_view}") - - def _get_all_views(self, conn): - from sqlalchemy import inspect - - return inspect(conn).get_view_names() - - class PandasSQLTest: """ Base class with common private methods for SQLAlchemy and fallback cases. @@ -1906,250 +1891,246 @@ def test_api_read_sql_duplicate_columns(conn, request): tm.assert_frame_equal(result, expected) -class _TestSQLApi(PandasSQLTest): - """ - Base class to test the public API. +@pytest.mark.parametrize("conn", all_connectable) +def test_read_table_columns(conn, request, test_frame1): + # test columns argument in read_table + conn_name = conn + conn = request.getfixturevalue(conn) + sql.to_sql(test_frame1, "test_frame", conn) - From this two classes are derived to run these tests for both the - sqlalchemy mode (`TestSQLApi`) and the fallback mode - (`TestSQLiteFallbackApi`). These tests are run with sqlite3. Specific - tests for the different sql flavours are included in `_TestSQLAlchemy`. + cols = ["A", "B"] - Notes: - flavor can always be passed even in SQLAlchemy mode, - should be correctly ignored. + if conn_name == "sqlite_buildin": + with pytest.raises(NotImplementedError, match=""): + sql.read_sql_table("test_frame", conn, columns=cols) + else: + result = sql.read_sql_table("test_frame", conn, columns=cols) + assert result.columns.tolist() == cols - we don't use drop_table because that isn't part of the public api - """ +@pytest.mark.parametrize("conn", all_connectable) +def test_read_table_index_col(conn, request, test_frame1): + # test columns argument in read_table + conn = request.getfixturevalue(conn) + sql.to_sql(test_frame1, "test_frame", conn) - flavor = "sqlite" - mode: str + result = sql.read_sql_table("test_frame", conn, index_col="index") + assert result.index.names == ["index"] - @pytest.fixture(autouse=True) - def setup_method(self, iris_path, types_data): - self.conn = self.connect() - self.load_iris_data(iris_path) - self.load_types_data(types_data) - self.load_test_data_and_sql() + result = sql.read_sql_table("test_frame", conn, index_col=["A", "B"]) + assert result.index.names == ["A", "B"] - def load_test_data_and_sql(self): - create_and_load_iris_view(self.conn) + result = sql.read_sql_table( + "test_frame", conn, index_col=["A", "B"], columns=["C", "D"] + ) + assert result.index.names == ["A", "B"] + assert result.columns.tolist() == ["C", "D"] -@pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="SQLAlchemy not installed") -class TestSQLApi(SQLAlchemyMixIn, _TestSQLApi): - """ - Test the public API as it would be used directly +@pytest.mark.parametrize("conn", all_connectable) +def test_read_sql_delegate(conn, request): + conn = request.getfixturevalue(conn) + iris_frame1 = sql.read_sql_query("SELECT * FROM iris", conn) + iris_frame2 = sql.read_sql("SELECT * FROM iris", conn) + tm.assert_frame_equal(iris_frame1, iris_frame2) - Tests for `read_sql_table` are included here, as this is specific for the - sqlalchemy mode. + iris_frame1 = sql.read_sql_table("iris", conn) + iris_frame2 = sql.read_sql("iris", conn) + tm.assert_frame_equal(iris_frame1, iris_frame2) - """ - flavor = "sqlite" - mode = "sqlalchemy" +@pytest.mark.parametrize("conn", all_connectable) +def test_not_reflect_all_tables(conn, request): + conn = request.getfixturevalue(conn) + from sqlalchemy import text + from sqlalchemy.engine import Engine - @classmethod - def setup_class(cls): - cls.engine = sqlalchemy.create_engine("sqlite:///:memory:") + # create invalid table + query_list = [ + text("CREATE TABLE invalid (x INTEGER, y UNKNOWN);"), + text("CREATE TABLE other_table (x INTEGER, y INTEGER);"), + ] + for query in query_list: + if isinstance(conn, Engine): + with conn.connect() as conn: + with conn.begin(): + conn.execute(query) + else: + with conn.begin(): + conn.execute(query) - def test_read_table_columns(self, test_frame1): - # test columns argument in read_table - sql.to_sql(test_frame1, "test_frame", self.conn) + with tm.assert_produces_warning(None): + sql.read_sql_table("other_table", conn) + sql.read_sql_query("SELECT * FROM other_table", conn) - cols = ["A", "B"] - result = sql.read_sql_table("test_frame", self.conn, columns=cols) - assert result.columns.tolist() == cols - def test_read_table_index_col(self, test_frame1): - # test columns argument in read_table - sql.to_sql(test_frame1, "test_frame", self.conn) +@pytest.mark.parametrize("conn", all_connectable) +def test_warning_case_insensitive_table_name(conn, request, test_frame1): + conn = request.getfixturevalue(conn) + # see gh-7815 + with tm.assert_produces_warning( + UserWarning, + match=( + r"The provided table name 'TABLE1' is not found exactly as such in " + r"the database after writing the table, possibly due to case " + r"sensitivity issues. Consider using lower case table names." + ), + ): + sql.SQLDatabase(conn).check_case_sensitive("TABLE1", "") - result = sql.read_sql_table("test_frame", self.conn, index_col="index") - assert result.index.names == ["index"] + # Test that the warning is certainly NOT triggered in a normal case. + with tm.assert_produces_warning(None): + test_frame1.to_sql(name="CaseSensitive", con=conn) - result = sql.read_sql_table("test_frame", self.conn, index_col=["A", "B"]) - assert result.index.names == ["A", "B"] - result = sql.read_sql_table( - "test_frame", self.conn, index_col=["A", "B"], columns=["C", "D"] - ) - assert result.index.names == ["A", "B"] - assert result.columns.tolist() == ["C", "D"] +@pytest.mark.parametrize("conn", all_connectable) +def _get_index_columns(conn, request, tbl_name): + conn = request.getfixturevalue(conn) + from sqlalchemy.engine import reflection - def test_read_sql_delegate(self): - iris_frame1 = sql.read_sql_query("SELECT * FROM iris", self.conn) - iris_frame2 = sql.read_sql("SELECT * FROM iris", self.conn) - tm.assert_frame_equal(iris_frame1, iris_frame2) + insp = reflection.Inspector.from_engine(conn) + ixs = insp.get_indexes("test_index_saved") + ixs = [i["column_names"] for i in ixs] + return ixs - iris_frame1 = sql.read_sql_table("iris", self.conn) - iris_frame2 = sql.read_sql("iris", self.conn) - tm.assert_frame_equal(iris_frame1, iris_frame2) - def test_not_reflect_all_tables(self): - from sqlalchemy import text - from sqlalchemy.engine import Engine +@pytest.mark.parametrize("conn", all_connectable) +def test_sqlalchemy_type_mapping(conn, request): + conn = request.getfixturevalue(conn) + from sqlalchemy import TIMESTAMP - # create invalid table - query_list = [ - text("CREATE TABLE invalid (x INTEGER, y UNKNOWN);"), - text("CREATE TABLE other_table (x INTEGER, y INTEGER);"), - ] - for query in query_list: - if isinstance(self.conn, Engine): - with self.conn.connect() as conn: - with conn.begin(): - conn.execute(query) - else: - with self.conn.begin(): - self.conn.execute(query) + # Test Timestamp objects (no datetime64 because of timezone) (GH9085) + df = DataFrame( + {"time": to_datetime(["2014-12-12 01:54", "2014-12-11 02:54"], utc=True)} + ) + db = sql.SQLDatabase(conn) + table = sql.SQLTable("test_type", db, frame=df) + # GH 9086: TIMESTAMP is the suggested type for datetimes with timezones + assert isinstance(table.table.c["time"].type, TIMESTAMP) - with tm.assert_produces_warning(None): - sql.read_sql_table("other_table", self.conn) - sql.read_sql_query("SELECT * FROM other_table", self.conn) - def test_warning_case_insensitive_table_name(self, test_frame1): - # see gh-7815 - with tm.assert_produces_warning( - UserWarning, - match=( - r"The provided table name 'TABLE1' is not found exactly as such in " - r"the database after writing the table, possibly due to case " - r"sensitivity issues. Consider using lower case table names." - ), - ): - sql.SQLDatabase(self.conn).check_case_sensitive("TABLE1", "") +@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize( + "integer, expected", + [ + ("int8", "SMALLINT"), + ("Int8", "SMALLINT"), + ("uint8", "SMALLINT"), + ("UInt8", "SMALLINT"), + ("int16", "SMALLINT"), + ("Int16", "SMALLINT"), + ("uint16", "INTEGER"), + ("UInt16", "INTEGER"), + ("int32", "INTEGER"), + ("Int32", "INTEGER"), + ("uint32", "BIGINT"), + ("UInt32", "BIGINT"), + ("int64", "BIGINT"), + ("Int64", "BIGINT"), + (int, "BIGINT" if np.dtype(int).name == "int64" else "INTEGER"), + ], +) +def test_sqlalchemy_integer_mapping(conn, request, integer, expected): + # GH35076 Map pandas integer to optimal SQLAlchemy integer type + conn = request.getfixturevalue(conn) + df = DataFrame([0, 1], columns=["a"], dtype=integer) + db = sql.SQLDatabase(conn) + table = sql.SQLTable("test_type", db, frame=df) - # Test that the warning is certainly NOT triggered in a normal case. - with tm.assert_produces_warning(None): - test_frame1.to_sql(name="CaseSensitive", con=self.conn) + result = str(table.table.c.a.type) + assert result == expected - def _get_index_columns(self, tbl_name): - from sqlalchemy.engine import reflection - insp = reflection.Inspector.from_engine(self.conn) - ixs = insp.get_indexes("test_index_saved") - ixs = [i["column_names"] for i in ixs] - return ixs +@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("integer", ["uint64", "UInt64"]) +def test_sqlalchemy_integer_overload_mapping(conn, request, integer): + conn = request.getfixturevalue(conn) + # GH35076 Map pandas integer to optimal SQLAlchemy integer type + df = DataFrame([0, 1], columns=["a"], dtype=integer) + db = sql.SQLDatabase(conn) + with pytest.raises( + ValueError, match="Unsigned 64 bit integer datatype is not supported" + ): + sql.SQLTable("test_type", db, frame=df) - def test_sqlalchemy_type_mapping(self): - from sqlalchemy import TIMESTAMP - # Test Timestamp objects (no datetime64 because of timezone) (GH9085) - df = DataFrame( - {"time": to_datetime(["2014-12-12 01:54", "2014-12-11 02:54"], utc=True)} - ) - db = sql.SQLDatabase(self.conn) - table = sql.SQLTable("test_type", db, frame=df) - # GH 9086: TIMESTAMP is the suggested type for datetimes with timezones - assert isinstance(table.table.c["time"].type, TIMESTAMP) +@pytest.mark.parametrize("conn", all_connectable) +def test_database_uri_string(conn, request, test_frame1): + conn = request.getfixturevalue(conn) + # Test read_sql and .to_sql method with a database URI (GH10654) + # db_uri = 'sqlite:///:memory:' # raises + # sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) near + # "iris": syntax error [SQL: 'iris'] + with tm.ensure_clean() as name: + db_uri = "sqlite:///" + name + table = "iris" + test_frame1.to_sql(name=table, con=db_uri, if_exists="replace", index=False) + test_frame2 = sql.read_sql(table, db_uri) + test_frame3 = sql.read_sql_table(table, db_uri) + query = "SELECT * FROM iris" + test_frame4 = sql.read_sql_query(query, db_uri) + tm.assert_frame_equal(test_frame1, test_frame2) + tm.assert_frame_equal(test_frame1, test_frame3) + tm.assert_frame_equal(test_frame1, test_frame4) + + +@td.skip_if_installed("pg8000") +@pytest.mark.parametrize("conn", all_connectable) +def test_pg8000_sqlalchemy_passthrough_error(conn, request): + conn = request.getfixturevalue(conn) + # using driver that will not be installed on CI to trigger error + # in sqlalchemy.create_engine -> test passing of this error to user + db_uri = "postgresql+pg8000://user:pass@host/dbname" + with pytest.raises(ImportError, match="pg8000"): + sql.read_sql("select * from table", db_uri) - @pytest.mark.parametrize( - "integer, expected", - [ - ("int8", "SMALLINT"), - ("Int8", "SMALLINT"), - ("uint8", "SMALLINT"), - ("UInt8", "SMALLINT"), - ("int16", "SMALLINT"), - ("Int16", "SMALLINT"), - ("uint16", "INTEGER"), - ("UInt16", "INTEGER"), - ("int32", "INTEGER"), - ("Int32", "INTEGER"), - ("uint32", "BIGINT"), - ("UInt32", "BIGINT"), - ("int64", "BIGINT"), - ("Int64", "BIGINT"), - (int, "BIGINT" if np.dtype(int).name == "int64" else "INTEGER"), - ], - ) - def test_sqlalchemy_integer_mapping(self, integer, expected): - # GH35076 Map pandas integer to optimal SQLAlchemy integer type - df = DataFrame([0, 1], columns=["a"], dtype=integer) - db = sql.SQLDatabase(self.conn) - table = sql.SQLTable("test_type", db, frame=df) - - result = str(table.table.c.a.type) - assert result == expected - - @pytest.mark.parametrize("integer", ["uint64", "UInt64"]) - def test_sqlalchemy_integer_overload_mapping(self, integer): - # GH35076 Map pandas integer to optimal SQLAlchemy integer type - df = DataFrame([0, 1], columns=["a"], dtype=integer) - db = sql.SQLDatabase(self.conn) - with pytest.raises( - ValueError, match="Unsigned 64 bit integer datatype is not supported" - ): - sql.SQLTable("test_type", db, frame=df) - def test_database_uri_string(self, test_frame1): - # Test read_sql and .to_sql method with a database URI (GH10654) - # db_uri = 'sqlite:///:memory:' # raises - # sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) near - # "iris": syntax error [SQL: 'iris'] - with tm.ensure_clean() as name: - db_uri = "sqlite:///" + name - table = "iris" - test_frame1.to_sql(name=table, con=db_uri, if_exists="replace", index=False) - test_frame2 = sql.read_sql(table, db_uri) - test_frame3 = sql.read_sql_table(table, db_uri) - query = "SELECT * FROM iris" - test_frame4 = sql.read_sql_query(query, db_uri) - tm.assert_frame_equal(test_frame1, test_frame2) - tm.assert_frame_equal(test_frame1, test_frame3) - tm.assert_frame_equal(test_frame1, test_frame4) - - @td.skip_if_installed("pg8000") - def test_pg8000_sqlalchemy_passthrough_error(self): - # using driver that will not be installed on CI to trigger error - # in sqlalchemy.create_engine -> test passing of this error to user - db_uri = "postgresql+pg8000://user:pass@host/dbname" - with pytest.raises(ImportError, match="pg8000"): - sql.read_sql("select * from table", db_uri) - - def test_query_by_text_obj(self): - # WIP : GH10846 - from sqlalchemy import text +@pytest.mark.parametrize("conn", all_connectable) +def test_query_by_text_obj(conn, request): + # WIP : GH10846 + conn = request.getfixturevalue(conn) + from sqlalchemy import text - name_text = text("select * from iris where name=:name") - iris_df = sql.read_sql(name_text, self.conn, params={"name": "Iris-versicolor"}) - all_names = set(iris_df["Name"]) - assert all_names == {"Iris-versicolor"} + name_text = text("select * from iris where name=:name") + iris_df = sql.read_sql(name_text, conn, params={"name": "Iris-versicolor"}) + all_names = set(iris_df["Name"]) + assert all_names == {"Iris-versicolor"} - def test_query_by_select_obj(self): - # WIP : GH10846 - from sqlalchemy import ( - bindparam, - select, - ) - iris = iris_table_metadata(self.flavor) - name_select = select(iris).where(iris.c.Name == bindparam("name")) - iris_df = sql.read_sql(name_select, self.conn, params={"name": "Iris-setosa"}) - all_names = set(iris_df["Name"]) - assert all_names == {"Iris-setosa"} +@pytest.mark.parametrize("conn", all_connectable) +def test_query_by_select_obj(conn, request): + conn = request.getfixturevalue(conn) + # WIP : GH10846 + from sqlalchemy import ( + bindparam, + select, + ) + + iris = iris_table_metadata(self.flavor) + name_select = select(iris).where(iris.c.Name == bindparam("name")) + iris_df = sql.read_sql(name_select, conn, params={"name": "Iris-setosa"}) + all_names = set(iris_df["Name"]) + assert all_names == {"Iris-setosa"} + - def test_column_with_percentage(self): - # GH 37157 - df = DataFrame({"A": [0, 1, 2], "%_variation": [3, 4, 5]}) - df.to_sql(name="test_column_percentage", con=self.conn, index=False) +@pytest.mark.parametrize("conn", all_connectable) +def test_column_with_percentage(conn, request): + # GH 37157 + conn = request.getfixturevalue(conn) + df = DataFrame({"A": [0, 1, 2], "%_variation": [3, 4, 5]}) + df.to_sql(name="test_column_percentage", con=conn, index=False) - res = sql.read_sql_table("test_column_percentage", self.conn) + res = sql.read_sql_table("test_column_percentage", conn) - tm.assert_frame_equal(res, df) + tm.assert_frame_equal(res, df) -class TestSQLiteFallbackApi(SQLiteMixIn, _TestSQLApi): +class TestSQLiteFallbackApi: """ Test the public sqlite connection fallback API """ - flavor = "sqlite" - mode = "fallback" - def connect(self, database=":memory:"): return sqlite3.connect(database) @@ -2230,7 +2211,7 @@ def test_sqlite_type_mapping(self): @pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="SQLAlchemy not installed") -class _TestSQLAlchemy(SQLAlchemyMixIn, PandasSQLTest): +class _TestSQLAlchemy: """ Base class for testing the sqlalchemy backend. @@ -3419,7 +3400,7 @@ def test_self_join_date_columns(self): # -- Test Sqlite / MySQL fallback -class TestSQLiteFallback(SQLiteMixIn, PandasSQLTest): +class TestSQLiteFallback: """ Test the fallback mode against an in-memory sqlite database. From 4c84f98ef5c07f34bde8cab13692e55889e6be41 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 8 Sep 2023 09:19:08 -0400 Subject: [PATCH 02/38] passing mixing class removal --- pandas/tests/io/test_sql.py | 51 ++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 75e1763cb1f51..fd29535325745 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1895,22 +1895,25 @@ def test_api_read_sql_duplicate_columns(conn, request): def test_read_table_columns(conn, request, test_frame1): # test columns argument in read_table conn_name = conn + if conn_name == "sqlite_buildin": + request.node.add_marker(pytest.mark.xfail(reason="Not Implemented")) + conn = request.getfixturevalue(conn) sql.to_sql(test_frame1, "test_frame", conn) cols = ["A", "B"] - if conn_name == "sqlite_buildin": - with pytest.raises(NotImplementedError, match=""): - sql.read_sql_table("test_frame", conn, columns=cols) - else: - result = sql.read_sql_table("test_frame", conn, columns=cols) - assert result.columns.tolist() == cols + result = sql.read_sql_table("test_frame", conn, columns=cols) + assert result.columns.tolist() == cols @pytest.mark.parametrize("conn", all_connectable) def test_read_table_index_col(conn, request, test_frame1): # test columns argument in read_table + conn_name = conn + if conn_name == "sqlite_buildin": + request.node.add_marker(pytest.mark.xfail(reason="Not Implemented")) + conn = request.getfixturevalue(conn) sql.to_sql(test_frame1, "test_frame", conn) @@ -1939,9 +1942,8 @@ def test_read_sql_delegate(conn, request): tm.assert_frame_equal(iris_frame1, iris_frame2) -@pytest.mark.parametrize("conn", all_connectable) -def test_not_reflect_all_tables(conn, request): - conn = request.getfixturevalue(conn) +def test_not_reflect_all_tables(sqlite_conn): + conn = sqlite_conn from sqlalchemy import text from sqlalchemy.engine import Engine @@ -1966,6 +1968,10 @@ def test_not_reflect_all_tables(conn, request): @pytest.mark.parametrize("conn", all_connectable) def test_warning_case_insensitive_table_name(conn, request, test_frame1): + conn_name = conn + if conn_name == "sqlite_buildin": + request.node.add_marker(pytest.mark.xfail(reason="Does not raise warning")) + conn = request.getfixturevalue(conn) # see gh-7815 with tm.assert_produces_warning( @@ -1983,17 +1989,6 @@ def test_warning_case_insensitive_table_name(conn, request, test_frame1): test_frame1.to_sql(name="CaseSensitive", con=conn) -@pytest.mark.parametrize("conn", all_connectable) -def _get_index_columns(conn, request, tbl_name): - conn = request.getfixturevalue(conn) - from sqlalchemy.engine import reflection - - insp = reflection.Inspector.from_engine(conn) - ixs = insp.get_indexes("test_index_saved") - ixs = [i["column_names"] for i in ixs] - return ixs - - @pytest.mark.parametrize("conn", all_connectable) def test_sqlalchemy_type_mapping(conn, request): conn = request.getfixturevalue(conn) @@ -2085,19 +2080,23 @@ def test_pg8000_sqlalchemy_passthrough_error(conn, request): sql.read_sql("select * from table", db_uri) -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_query_by_text_obj(conn, request): # WIP : GH10846 + conn_name = conn conn = request.getfixturevalue(conn) from sqlalchemy import text - name_text = text("select * from iris where name=:name") + if "postgres" in conn_name: + name_text = text('select * from iris where "Name"=:name') + else: + name_text = text("select * from iris where name=:name") iris_df = sql.read_sql(name_text, conn, params={"name": "Iris-versicolor"}) all_names = set(iris_df["Name"]) assert all_names == {"Iris-versicolor"} -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_query_by_select_obj(conn, request): conn = request.getfixturevalue(conn) # WIP : GH10846 @@ -2106,7 +2105,7 @@ def test_query_by_select_obj(conn, request): select, ) - iris = iris_table_metadata(self.flavor) + iris = iris_table_metadata("postgres") name_select = select(iris).where(iris.c.Name == bindparam("name")) iris_df = sql.read_sql(name_select, conn, params={"name": "Iris-setosa"}) all_names = set(iris_df["Name"]) @@ -2116,6 +2115,10 @@ def test_query_by_select_obj(conn, request): @pytest.mark.parametrize("conn", all_connectable) def test_column_with_percentage(conn, request): # GH 37157 + conn_name = conn + if conn_name == "sqlite_buildin": + request.node.add_marker(pytest.mark.xfail(reason="Not Implemented")) + conn = request.getfixturevalue(conn) df = DataFrame({"A": [0, 1, 2], "%_variation": [3, 4, 5]}) df.to_sql(name="test_column_percentage", con=conn, index=False) From eada6f657e1873303d3a415414ed6ba56a05a746 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 8 Sep 2023 10:50:21 -0400 Subject: [PATCH 03/38] converted non-sqlalchemy tests --- pandas/tests/io/test_sql.py | 131 +++++++++++++++++------------------- 1 file changed, 60 insertions(+), 71 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index fd29535325745..9ad0435087901 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2128,85 +2128,74 @@ def test_column_with_percentage(conn, request): tm.assert_frame_equal(res, df) -class TestSQLiteFallbackApi: - """ - Test the public sqlite connection fallback API +def test_sql_open_close(test_frame3): + # Test if the IO in the database still work if the connection closed + # between the writing and reading (as in many real situations). - """ + with tm.ensure_clean() as name: + with closing(sqlite3.connect(name)) as conn: + assert sql.to_sql(test_frame3, "test_frame3_legacy", conn, index=False) == 4 - def connect(self, database=":memory:"): - return sqlite3.connect(database) + with closing(sqlite3.connect(name)) as conn: + result = sql.read_sql_query("SELECT * FROM test_frame3_legacy;", conn) - def test_sql_open_close(self, test_frame3): - # Test if the IO in the database still work if the connection closed - # between the writing and reading (as in many real situations). + tm.assert_frame_equal(test_frame3, result) - with tm.ensure_clean() as name: - with closing(self.connect(name)) as conn: - assert ( - sql.to_sql(test_frame3, "test_frame3_legacy", conn, index=False) - == 4 - ) - with closing(self.connect(name)) as conn: - result = sql.read_sql_query("SELECT * FROM test_frame3_legacy;", conn) +@pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") +def test_con_string_import_error(): + conn = "mysql://root@localhost/pandas" + msg = "Using URI string without sqlalchemy installed" + with pytest.raises(ImportError, match=msg): + sql.read_sql("SELECT * FROM iris", conn) - tm.assert_frame_equal(test_frame3, result) - @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") - def test_con_string_import_error(self): - conn = "mysql://root@localhost/pandas" - msg = "Using URI string without sqlalchemy installed" - with pytest.raises(ImportError, match=msg): - sql.read_sql("SELECT * FROM iris", conn) +@pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") +def test_con_unknown_dbapi2_class_does_not_error_without_sql_alchemy_installed(): + class MockSqliteConnection: + def __init__(self, *args, **kwargs) -> None: + self.conn = sqlite3.Connection(*args, **kwargs) - @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") - def test_con_unknown_dbapi2_class_does_not_error_without_sql_alchemy_installed( - self, - ): - class MockSqliteConnection: - def __init__(self, *args, **kwargs) -> None: - self.conn = sqlite3.Connection(*args, **kwargs) - - def __getattr__(self, name): - return getattr(self.conn, name) - - def close(self): - self.conn.close() - - with contextlib.closing(MockSqliteConnection(":memory:")) as conn: - with tm.assert_produces_warning(UserWarning): - sql.read_sql("SELECT 1", conn) - - def test_read_sql_delegate(self): - iris_frame1 = sql.read_sql_query("SELECT * FROM iris", self.conn) - iris_frame2 = sql.read_sql("SELECT * FROM iris", self.conn) - tm.assert_frame_equal(iris_frame1, iris_frame2) - - msg = "Execution failed on sql 'iris': near \"iris\": syntax error" - with pytest.raises(sql.DatabaseError, match=msg): - sql.read_sql("iris", self.conn) - - def test_get_schema2(self, test_frame1): - # without providing a connection object (available for backwards comp) - create_sql = sql.get_schema(test_frame1, "test") - assert "CREATE" in create_sql - - def _get_sqlite_column_type(self, schema, column): - for col in schema.split("\n"): - if col.split()[0].strip('"') == column: - return col.split()[1] - raise ValueError(f"Column {column} not found") - - def test_sqlite_type_mapping(self): - # Test Timestamp objects (no datetime64 because of timezone) (GH9085) - df = DataFrame( - {"time": to_datetime(["2014-12-12 01:54", "2014-12-11 02:54"], utc=True)} - ) - db = sql.SQLiteDatabase(self.conn) - table = sql.SQLiteTable("test_type", db, frame=df) - schema = table.sql_schema() - assert self._get_sqlite_column_type(schema, "time") == "TIMESTAMP" + def __getattr__(self, name): + return getattr(self.conn, name) + + def close(self): + self.conn.close() + + with contextlib.closing(MockSqliteConnection(":memory:")) as conn: + with tm.assert_produces_warning(UserWarning): + sql.read_sql("SELECT 1", conn) + + +def test_read_sql_delegate(sqlite_buildin_iris): + conn = sqlite_buildin_iris + iris_frame1 = sql.read_sql_query("SELECT * FROM iris", conn) + iris_frame2 = sql.read_sql("SELECT * FROM iris", conn) + tm.assert_frame_equal(iris_frame1, iris_frame2) + + msg = "Execution failed on sql 'iris': near \"iris\": syntax error" + with pytest.raises(sql.DatabaseError, match=msg): + sql.read_sql("iris", conn) + + +def test_get_schema2(test_frame1): + # without providing a connection object (available for backwards comp) + create_sql = sql.get_schema(test_frame1, "test") + assert "CREATE" in create_sql + + +def test_sqlite_type_mapping(sqlite_buildin): + # Test Timestamp objects (no datetime64 because of timezone) (GH9085) + conn = sqlite_buildin + df = DataFrame( + {"time": to_datetime(["2014-12-12 01:54", "2014-12-11 02:54"], utc=True)} + ) + db = sql.SQLiteDatabase(conn) + table = sql.SQLiteTable("test_type", db, frame=df) + schema = table.sql_schema() + for col in schema.split("\n"): + if col.split()[0].strip('"') == "time": + assert col.split()[1] == "TIMESTAMP" # ----------------------------------------------------------------------------- From 8dc41b0afef520d57bbd773b4c6cc38228d3867d Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 8 Sep 2023 16:37:54 -0400 Subject: [PATCH 04/38] large refactor --- pandas/tests/io/test_sql.py | 1661 ++++++++++++++++++++--------------- 1 file changed, 972 insertions(+), 689 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 9ad0435087901..7a1f1089b11bc 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1891,6 +1891,7 @@ def test_api_read_sql_duplicate_columns(conn, request): tm.assert_frame_equal(result, expected) +@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_read_table_columns(conn, request, test_frame1): # test columns argument in read_table @@ -1907,6 +1908,7 @@ def test_read_table_columns(conn, request, test_frame1): assert result.columns.tolist() == cols +@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_read_table_index_col(conn, request, test_frame1): # test columns argument in read_table @@ -1930,6 +1932,7 @@ def test_read_table_index_col(conn, request, test_frame1): assert result.columns.tolist() == ["C", "D"] +@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_read_sql_delegate(conn, request): conn = request.getfixturevalue(conn) @@ -1966,6 +1969,7 @@ def test_not_reflect_all_tables(sqlite_conn): sql.read_sql_query("SELECT * FROM other_table", conn) +@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_warning_case_insensitive_table_name(conn, request, test_frame1): conn_name = conn @@ -1989,6 +1993,7 @@ def test_warning_case_insensitive_table_name(conn, request, test_frame1): test_frame1.to_sql(name="CaseSensitive", con=conn) +@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_sqlalchemy_type_mapping(conn, request): conn = request.getfixturevalue(conn) @@ -2004,6 +2009,7 @@ def test_sqlalchemy_type_mapping(conn, request): assert isinstance(table.table.c["time"].type, TIMESTAMP) +@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) @pytest.mark.parametrize( "integer, expected", @@ -2036,6 +2042,7 @@ def test_sqlalchemy_integer_mapping(conn, request, integer, expected): assert result == expected +@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) @pytest.mark.parametrize("integer", ["uint64", "UInt64"]) def test_sqlalchemy_integer_overload_mapping(conn, request, integer): @@ -2049,6 +2056,7 @@ def test_sqlalchemy_integer_overload_mapping(conn, request, integer): sql.SQLTable("test_type", db, frame=df) +@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_database_uri_string(conn, request, test_frame1): conn = request.getfixturevalue(conn) @@ -2070,6 +2078,7 @@ def test_database_uri_string(conn, request, test_frame1): @td.skip_if_installed("pg8000") +@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_pg8000_sqlalchemy_passthrough_error(conn, request): conn = request.getfixturevalue(conn) @@ -2080,6 +2089,7 @@ def test_pg8000_sqlalchemy_passthrough_error(conn, request): sql.read_sql("select * from table", db_uri) +@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_query_by_text_obj(conn, request): # WIP : GH10846 @@ -2096,6 +2106,7 @@ def test_query_by_text_obj(conn, request): assert all_names == {"Iris-versicolor"} +@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_query_by_select_obj(conn, request): conn = request.getfixturevalue(conn) @@ -2112,6 +2123,7 @@ def test_query_by_select_obj(conn, request): assert all_names == {"Iris-setosa"} +@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_column_with_percentage(conn, request): # GH 37157 @@ -2128,6 +2140,7 @@ def test_column_with_percentage(conn, request): tm.assert_frame_equal(res, df) +@pytest.mark.db def test_sql_open_close(test_frame3): # Test if the IO in the database still work if the connection closed # between the writing and reading (as in many real situations). @@ -2142,6 +2155,7 @@ def test_sql_open_close(test_frame3): tm.assert_frame_equal(test_frame3, result) +@pytest.mark.db @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") def test_con_string_import_error(): conn = "mysql://root@localhost/pandas" @@ -2150,6 +2164,7 @@ def test_con_string_import_error(): sql.read_sql("SELECT * FROM iris", conn) +@pytest.mark.db @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") def test_con_unknown_dbapi2_class_does_not_error_without_sql_alchemy_installed(): class MockSqliteConnection: @@ -2167,6 +2182,7 @@ def close(self): sql.read_sql("SELECT 1", conn) +@pytest.mark.db def test_read_sql_delegate(sqlite_buildin_iris): conn = sqlite_buildin_iris iris_frame1 = sql.read_sql_query("SELECT * FROM iris", conn) @@ -2178,12 +2194,14 @@ def test_read_sql_delegate(sqlite_buildin_iris): sql.read_sql("iris", conn) +@pytest.mark.db def test_get_schema2(test_frame1): # without providing a connection object (available for backwards comp) create_sql = sql.get_schema(test_frame1, "test") assert "CREATE" in create_sql +@pytest.mark.db def test_sqlite_type_mapping(sqlite_buildin): # Test Timestamp objects (no datetime64 because of timezone) (GH9085) conn = sqlite_buildin @@ -2202,779 +2220,1027 @@ def test_sqlite_type_mapping(sqlite_buildin): # -- Database flavor specific tests -@pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="SQLAlchemy not installed") -class _TestSQLAlchemy: - """ - Base class for testing the sqlalchemy backend. +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_create_table(conn, request): + if conn == "sqlite_str": + pytest.skip("sqlite_str has no inspection system") - Subclasses for specific database types are created below. Tests that - deviate for each flavor are overwritten there. + from sqlalchemy import inspect - """ + conn = request.getfixturevalue(conn) - flavor: str + temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) + with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: + assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 - @classmethod - def setup_class(cls): - cls.setup_driver() - cls.setup_engine() + insp = inspect(conn) + assert insp.has_table("temp_frame") - @pytest.fixture(autouse=True) - def setup_method(self, iris_path, types_data): - try: - self.conn = self.engine.connect() - self.pandasSQL = sql.SQLDatabase(self.conn) - except sqlalchemy.exc.OperationalError: - pytest.skip(f"Can't connect to {self.flavor} server") - self.load_iris_data(iris_path) - self.load_types_data(types_data) + # Cleanup + with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: + pandasSQL.drop_table("temp_frame") - @classmethod - def setup_driver(cls): - raise NotImplementedError() - @classmethod - def setup_engine(cls): - raise NotImplementedError() +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_drop_table(conn, request): + if conn == "sqlite_str": + pytest.skip("sqlite_str has no inspection system") + elif "engine" in conn: + request.node.add_marker( + pytest.xfail(reason="fails and hangs forever with engine") + ) - def test_read_sql_parameter(self, sql_strings): - self._read_sql_iris_parameter(sql_strings) + from sqlalchemy import inspect - def test_read_sql_named_parameter(self, sql_strings): - self._read_sql_iris_named_parameter(sql_strings) + conn = request.getfixturevalue(conn) - def test_to_sql_empty(self, test_frame1): - self._to_sql_empty(test_frame1) + temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) + pandasSQL = sql.SQLDatabase(conn) + assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 - def test_create_table(self): - from sqlalchemy import inspect + insp = inspect(conn) + assert insp.has_table("temp_frame") - temp_conn = self.connect() - temp_frame = DataFrame( - {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} + pandasSQL.drop_table("temp_frame") + try: + insp.clear_cache() # needed with SQLAlchemy 2.0, unavailable prior + except AttributeError: + pass + assert not insp.has_table("temp_frame") + + +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +def test_roundtrip(conn, request, test_frame1): + conn_name = conn + if conn == "sqlite_str": + pytest.skip("sqlite_str has no inspection system") + elif "engine" in conn: + request.node.add_marker( + pytest.xfail(reason="fails and hangs forever with engine") ) - with sql.SQLDatabase(temp_conn, need_transaction=True) as pandasSQL: - assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 - insp = inspect(temp_conn) - assert insp.has_table("temp_frame") + conn = request.getfixturevalue(conn) + drop_table("test_frame_roundtrip", conn) - # Cleanup - with sql.SQLDatabase(temp_conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("temp_frame") + if conn_name == "sqlite_buildin": + pandasSQL = sql.SQLiteDatabase(conn) + else: + pandasSQL = sql.SQLDatabase(conn) + assert pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4 + result = pandasSQL.read_query("SELECT * FROM test_frame_roundtrip") - def test_drop_table(self): - from sqlalchemy import inspect + result.set_index("level_0", inplace=True) + # result.index.astype(int) - temp_conn = self.connect() - temp_frame = DataFrame( - {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} - ) - pandasSQL = sql.SQLDatabase(temp_conn) - assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 + result.index.name = None - insp = inspect(temp_conn) - assert insp.has_table("temp_frame") + tm.assert_frame_equal(result, test_frame1) - pandasSQL.drop_table("temp_frame") - try: - insp.clear_cache() # needed with SQLAlchemy 2.0, unavailable prior - except AttributeError: - pass - assert not insp.has_table("temp_frame") - def test_roundtrip(self, test_frame1): - self._roundtrip(test_frame1) +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable_iris) +def test_execute_sql(conn, request): + conn_name = conn + conn = request.getfixturevalue(conn) + if conn_name == "sqlite_buildin_iris": + pandasSQL = sql.SQLiteDatabase(conn) + else: + pandasSQL = sql.SQLDatabase(conn) + iris_results = pandasSQL.execute("SELECT * FROM iris") + row = iris_results.fetchone() + tm.equalContents(row, [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]) - def test_execute_sql(self): - self._execute_sql() - def test_read_table(self): - iris_frame = sql.read_sql_table("iris", con=self.conn) - check_iris_frame(iris_frame) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) +def test_read_table(conn, request): + conn = request.getfixturevalue(conn) + iris_frame = sql.read_sql_table("iris", con=conn) + check_iris_frame(iris_frame) - def test_read_table_columns(self): - iris_frame = sql.read_sql_table( - "iris", con=self.conn, columns=["SepalLength", "SepalLength"] - ) - tm.equalContents(iris_frame.columns.values, ["SepalLength", "SepalLength"]) - def test_read_table_absent_raises(self): - msg = "Table this_doesnt_exist not found" - with pytest.raises(ValueError, match=msg): - sql.read_sql_table("this_doesnt_exist", con=self.conn) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) +def test_read_table_columns(conn, request): + conn = request.getfixturevalue(conn) + iris_frame = sql.read_sql_table( + "iris", con=conn, columns=["SepalLength", "SepalLength"] + ) + tm.equalContents(iris_frame.columns.values, ["SepalLength", "SepalLength"]) - def test_default_type_conversion(self): - df = sql.read_sql_table("types", self.conn) - assert issubclass(df.FloatCol.dtype.type, np.floating) - assert issubclass(df.IntCol.dtype.type, np.integer) - assert issubclass(df.BoolCol.dtype.type, np.bool_) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) +def test_read_table_absent_raises(conn, request): + conn = request.getfixturevalue(conn) + msg = "Table this_doesnt_exist not found" + with pytest.raises(ValueError, match=msg): + sql.read_sql_table("this_doesnt_exist", con=conn) - # Int column with NA values stays as float - assert issubclass(df.IntColWithNull.dtype.type, np.floating) - # Bool column with NA values becomes object - assert issubclass(df.BoolColWithNull.dtype.type, object) - def test_bigint(self): - # int64 should be converted to BigInteger, GH7433 - df = DataFrame(data={"i64": [2**62]}) - assert df.to_sql(name="test_bigint", con=self.conn, index=False) == 1 - result = sql.read_sql_table("test_bigint", self.conn) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_default_type_conversion(conn, request): + conn_name = conn + if conn_name == "sqlite_str": + pytest.skip("types tables not created in sqlite_str fixture") + elif "mysql" in conn_name or "sqlite" in conn_name: + request.node.add_marker( + pytest.xfail(reason="boolean dtype not inferred properly") + ) - tm.assert_frame_equal(df, result) + conn = request.getfixturevalue(conn) + df = sql.read_sql_table("types", conn) - def test_default_date_load(self): - df = sql.read_sql_table("types", self.conn) + assert issubclass(df.FloatCol.dtype.type, np.floating) + assert issubclass(df.IntCol.dtype.type, np.integer) + assert issubclass(df.BoolCol.dtype.type, np.bool_) - # IMPORTANT - sqlite has no native date type, so shouldn't parse, but - # MySQL SHOULD be converted. - assert issubclass(df.DateCol.dtype.type, np.datetime64) - - def test_datetime_with_timezone(self, request): - # edge case that converts postgresql datetime with time zone types - # to datetime64[ns,psycopg2.tz.FixedOffsetTimezone..], which is ok - # but should be more natural, so coerce to datetime64[ns] for now - - def check(col): - # check that a column is either datetime64[ns] - # or datetime64[ns, UTC] - if lib.is_np_dtype(col.dtype, "M"): - # "2000-01-01 00:00:00-08:00" should convert to - # "2000-01-01 08:00:00" - assert col[0] == Timestamp("2000-01-01 08:00:00") - - # "2000-06-01 00:00:00-07:00" should convert to - # "2000-06-01 07:00:00" - assert col[1] == Timestamp("2000-06-01 07:00:00") - - elif isinstance(col.dtype, DatetimeTZDtype): - assert str(col.dt.tz) == "UTC" - - # "2000-01-01 00:00:00-08:00" should convert to - # "2000-01-01 08:00:00" - # "2000-06-01 00:00:00-07:00" should convert to - # "2000-06-01 07:00:00" - # GH 6415 - expected_data = [ - Timestamp("2000-01-01 08:00:00", tz="UTC"), - Timestamp("2000-06-01 07:00:00", tz="UTC"), - ] - expected = Series(expected_data, name=col.name) - tm.assert_series_equal(col, expected) + # Int column with NA values stays as float + assert issubclass(df.IntColWithNull.dtype.type, np.floating) + # Bool column with NA values becomes object + assert issubclass(df.BoolColWithNull.dtype.type, object) - else: - raise AssertionError( - f"DateCol loaded with incorrect type -> {col.dtype}" - ) - # GH11216 - df = read_sql_query("select * from types", self.conn) - if not hasattr(df, "DateColWithTz"): - request.node.add_marker( - pytest.mark.xfail(reason="no column with datetime with time zone") - ) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_bigint(conn, request): + # int64 should be converted to BigInteger, GH7433 + conn = request.getfixturevalue(conn) + df = DataFrame(data={"i64": [2**62]}) + assert df.to_sql(name="test_bigint", con=conn, index=False) == 1 + result = sql.read_sql_table("test_bigint", conn) - # this is parsed on Travis (linux), but not on macosx for some reason - # even with the same versions of psycopg2 & sqlalchemy, possibly a - # Postgresql server version difference - col = df.DateColWithTz - assert isinstance(col.dtype, DatetimeTZDtype) + tm.assert_frame_equal(df, result) - df = read_sql_query( - "select * from types", self.conn, parse_dates=["DateColWithTz"] - ) - if not hasattr(df, "DateColWithTz"): - request.node.add_marker( - pytest.mark.xfail(reason="no column with datetime with time zone") - ) - col = df.DateColWithTz - assert isinstance(col.dtype, DatetimeTZDtype) - assert str(col.dt.tz) == "UTC" - check(df.DateColWithTz) - - df = concat( - list(read_sql_query("select * from types", self.conn, chunksize=1)), - ignore_index=True, - ) - col = df.DateColWithTz - assert isinstance(col.dtype, DatetimeTZDtype) - assert str(col.dt.tz) == "UTC" - expected = sql.read_sql_table("types", self.conn) - col = expected.DateColWithTz - assert isinstance(col.dtype, DatetimeTZDtype) - tm.assert_series_equal(df.DateColWithTz, expected.DateColWithTz) - - # xref #7139 - # this might or might not be converted depending on the postgres driver - df = sql.read_sql_table("types", self.conn) - check(df.DateColWithTz) - def test_datetime_with_timezone_roundtrip(self): - # GH 9086 - # Write datetimetz data to a db and read it back - # For dbs that support timestamps with timezones, should get back UTC - # otherwise naive data should be returned - expected = DataFrame( - {"A": date_range("2013-01-01 09:00:00", periods=3, tz="US/Pacific")} +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_default_date_load(conn, request): + conn_name = conn + if conn_name == "sqlite_str": + pytest.skip("types tables not created in sqlite_str fixture") + elif "sqlite" in conn_name: + request.node.add_marker( + pytest.xfail(reason="sqlite does not read date properly") ) - assert expected.to_sql(name="test_datetime_tz", con=self.conn, index=False) == 3 - if self.flavor == "postgresql": - # SQLAlchemy "timezones" (i.e. offsets) are coerced to UTC - expected["A"] = expected["A"].dt.tz_convert("UTC") - else: - # Otherwise, timestamps are returned as local, naive - expected["A"] = expected["A"].dt.tz_localize(None) + conn = request.getfixturevalue(conn) + df = sql.read_sql_table("types", conn) - result = sql.read_sql_table("test_datetime_tz", self.conn) - tm.assert_frame_equal(result, expected) + assert issubclass(df.DateCol.dtype.type, np.datetime64) - result = sql.read_sql_query("SELECT * FROM test_datetime_tz", self.conn) - if self.flavor == "sqlite": - # read_sql_query does not return datetime type like read_sql_table - assert isinstance(result.loc[0, "A"], str) - result["A"] = to_datetime(result["A"]) - tm.assert_frame_equal(result, expected) - def test_out_of_bounds_datetime(self): - # GH 26761 - data = DataFrame({"date": datetime(9999, 1, 1)}, index=[0]) - assert data.to_sql(name="test_datetime_obb", con=self.conn, index=False) == 1 - result = sql.read_sql_table("test_datetime_obb", self.conn) - expected = DataFrame([pd.NaT], columns=["date"]) - tm.assert_frame_equal(result, expected) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) +def test_datetime_with_timezone(conn, request): + # edge case that converts postgresql datetime with time zone types + # to datetime64[ns,psycopg2.tz.FixedOffsetTimezone..], which is ok + # but should be more natural, so coerce to datetime64[ns] for now + + def check(col): + # check that a column is either datetime64[ns] + # or datetime64[ns, UTC] + if lib.is_np_dtype(col.dtype, "M"): + # "2000-01-01 00:00:00-08:00" should convert to + # "2000-01-01 08:00:00" + assert col[0] == Timestamp("2000-01-01 08:00:00") + + # "2000-06-01 00:00:00-07:00" should convert to + # "2000-06-01 07:00:00" + assert col[1] == Timestamp("2000-06-01 07:00:00") + + elif isinstance(col.dtype, DatetimeTZDtype): + assert str(col.dt.tz) == "UTC" + + # "2000-01-01 00:00:00-08:00" should convert to + # "2000-01-01 08:00:00" + # "2000-06-01 00:00:00-07:00" should convert to + # "2000-06-01 07:00:00" + # GH 6415 + expected_data = [ + Timestamp("2000-01-01 08:00:00", tz="UTC"), + Timestamp("2000-06-01 07:00:00", tz="UTC"), + ] + expected = Series(expected_data, name=col.name) + tm.assert_series_equal(col, expected) - def test_naive_datetimeindex_roundtrip(self): - # GH 23510 - # Ensure that a naive DatetimeIndex isn't converted to UTC - dates = date_range("2018-01-01", periods=5, freq="6H")._with_freq(None) - expected = DataFrame({"nums": range(5)}, index=dates) - assert ( - expected.to_sql(name="foo_table", con=self.conn, index_label="info_date") - == 5 - ) - result = sql.read_sql_table("foo_table", self.conn, index_col="info_date") - # result index with gain a name from a set_index operation; expected - tm.assert_frame_equal(result, expected, check_names=False) + else: + raise AssertionError(f"DateCol loaded with incorrect type -> {col.dtype}") - def test_date_parsing(self): - # No Parsing - df = sql.read_sql_table("types", self.conn) - expected_type = object if self.flavor == "sqlite" else np.datetime64 - assert issubclass(df.DateCol.dtype.type, expected_type) + # GH11216 + conn = request.getfixturevalue(conn) + df = read_sql_query("select * from types", conn) + if not hasattr(df, "DateColWithTz"): + request.node.add_marker( + pytest.mark.xfail(reason="no column with datetime with time zone") + ) - df = sql.read_sql_table("types", self.conn, parse_dates=["DateCol"]) - assert issubclass(df.DateCol.dtype.type, np.datetime64) + # this is parsed on Travis (linux), but not on macosx for some reason + # even with the same versions of psycopg2 & sqlalchemy, possibly a + # Postgresql server version difference + col = df.DateColWithTz + assert isinstance(col.dtype, DatetimeTZDtype) - df = sql.read_sql_table( - "types", self.conn, parse_dates={"DateCol": "%Y-%m-%d %H:%M:%S"} + df = read_sql_query("select * from types", conn, parse_dates=["DateColWithTz"]) + if not hasattr(df, "DateColWithTz"): + request.node.add_marker( + pytest.mark.xfail(reason="no column with datetime with time zone") ) - assert issubclass(df.DateCol.dtype.type, np.datetime64) + col = df.DateColWithTz + assert isinstance(col.dtype, DatetimeTZDtype) + assert str(col.dt.tz) == "UTC" + check(df.DateColWithTz) + + df = concat( + list(read_sql_query("select * from types", conn, chunksize=1)), + ignore_index=True, + ) + col = df.DateColWithTz + assert isinstance(col.dtype, DatetimeTZDtype) + assert str(col.dt.tz) == "UTC" + expected = sql.read_sql_table("types", conn) + col = expected.DateColWithTz + assert isinstance(col.dtype, DatetimeTZDtype) + tm.assert_series_equal(df.DateColWithTz, expected.DateColWithTz) + + # xref #7139 + # this might or might not be converted depending on the postgres driver + df = sql.read_sql_table("types", conn) + check(df.DateColWithTz) - df = sql.read_sql_table( - "types", - self.conn, - parse_dates={"DateCol": {"format": "%Y-%m-%d %H:%M:%S"}}, - ) - assert issubclass(df.DateCol.dtype.type, np.datetime64) - df = sql.read_sql_table("types", self.conn, parse_dates=["IntDateCol"]) - assert issubclass(df.IntDateCol.dtype.type, np.datetime64) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_datetime_with_timezone_roundtrip(conn, request): + conn_name = conn + conn = request.getfixturevalue(conn) + # GH 9086 + # Write datetimetz data to a db and read it back + # For dbs that support timestamps with timezones, should get back UTC + # otherwise naive data should be returned + expected = DataFrame( + {"A": date_range("2013-01-01 09:00:00", periods=3, tz="US/Pacific")} + ) + assert expected.to_sql(name="test_datetime_tz", con=conn, index=False) == 3 - df = sql.read_sql_table("types", self.conn, parse_dates={"IntDateCol": "s"}) - assert issubclass(df.IntDateCol.dtype.type, np.datetime64) + if "postgresql" in conn_name: + # SQLAlchemy "timezones" (i.e. offsets) are coerced to UTC + expected["A"] = expected["A"].dt.tz_convert("UTC") + else: + # Otherwise, timestamps are returned as local, naive + expected["A"] = expected["A"].dt.tz_localize(None) - df = sql.read_sql_table( - "types", self.conn, parse_dates={"IntDateCol": {"unit": "s"}} - ) - assert issubclass(df.IntDateCol.dtype.type, np.datetime64) + result = sql.read_sql_table("test_datetime_tz", conn) + tm.assert_frame_equal(result, expected) - def test_datetime(self): - df = DataFrame( - {"A": date_range("2013-01-01 09:00:00", periods=3), "B": np.arange(3.0)} - ) - assert df.to_sql(name="test_datetime", con=self.conn) == 3 + result = sql.read_sql_query("SELECT * FROM test_datetime_tz", conn) + if "sqlite" in conn_name: + # read_sql_query does not return datetime type like read_sql_table + assert isinstance(result.loc[0, "A"], str) + result["A"] = to_datetime(result["A"]) + tm.assert_frame_equal(result, expected) - # with read_table -> type information from schema used - result = sql.read_sql_table("test_datetime", self.conn) - result = result.drop("index", axis=1) - tm.assert_frame_equal(result, df) - # with read_sql -> no type information -> sqlite has no native - result = sql.read_sql_query("SELECT * FROM test_datetime", self.conn) - result = result.drop("index", axis=1) - if self.flavor == "sqlite": - assert isinstance(result.loc[0, "A"], str) - result["A"] = to_datetime(result["A"]) - tm.assert_frame_equal(result, df) - else: - tm.assert_frame_equal(result, df) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_out_of_bounds_datetime(conn, request): + # GH 26761 + conn = request.getfixturevalue(conn) + data = DataFrame({"date": datetime(9999, 1, 1)}, index=[0]) + assert data.to_sql(name="test_datetime_obb", con=conn, index=False) == 1 + result = sql.read_sql_table("test_datetime_obb", conn) + expected = DataFrame([pd.NaT], columns=["date"]) + tm.assert_frame_equal(result, expected) - def test_datetime_NaT(self): - df = DataFrame( - {"A": date_range("2013-01-01 09:00:00", periods=3), "B": np.arange(3.0)} - ) - df.loc[1, "A"] = np.nan - assert df.to_sql(name="test_datetime", con=self.conn, index=False) == 3 - # with read_table -> type information from schema used - result = sql.read_sql_table("test_datetime", self.conn) - tm.assert_frame_equal(result, df) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_naive_datetimeindex_roundtrip(conn, request): + # GH 23510 + # Ensure that a naive DatetimeIndex isn't converted to UTC + conn = request.getfixturevalue(conn) + dates = date_range("2018-01-01", periods=5, freq="6H")._with_freq(None) + expected = DataFrame({"nums": range(5)}, index=dates) + assert expected.to_sql(name="foo_table", con=conn, index_label="info_date") == 5 + result = sql.read_sql_table("foo_table", conn, index_col="info_date") + # result index with gain a name from a set_index operation; expected + tm.assert_frame_equal(result, expected, check_names=False) - # with read_sql -> no type information -> sqlite has no native - result = sql.read_sql_query("SELECT * FROM test_datetime", self.conn) - if self.flavor == "sqlite": - assert isinstance(result.loc[0, "A"], str) - result["A"] = to_datetime(result["A"], errors="coerce") - tm.assert_frame_equal(result, df) - else: - tm.assert_frame_equal(result, df) - def test_datetime_date(self): - # test support for datetime.date - df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"]) - assert df.to_sql(name="test_date", con=self.conn, index=False) == 2 - res = read_sql_table("test_date", self.conn) - result = res["a"] - expected = to_datetime(df["a"]) - # comes back as datetime64 - tm.assert_series_equal(result, expected) - - def test_datetime_time(self, sqlite_buildin): - # test support for datetime.time - df = DataFrame([time(9, 0, 0), time(9, 1, 30)], columns=["a"]) - assert df.to_sql(name="test_time", con=self.conn, index=False) == 2 - res = read_sql_table("test_time", self.conn) - tm.assert_frame_equal(res, df) - - # GH8341 - # first, use the fallback to have the sqlite adapter put in place - sqlite_conn = sqlite_buildin - assert sql.to_sql(df, "test_time2", sqlite_conn, index=False) == 2 - res = sql.read_sql_query("SELECT * FROM test_time2", sqlite_conn) - ref = df.map(lambda _: _.strftime("%H:%M:%S.%f")) - tm.assert_frame_equal(ref, res) # check if adapter is in place - # then test if sqlalchemy is unaffected by the sqlite adapter - assert sql.to_sql(df, "test_time3", self.conn, index=False) == 2 - if self.flavor == "sqlite": - res = sql.read_sql_query("SELECT * FROM test_time3", self.conn) - ref = df.map(lambda _: _.strftime("%H:%M:%S.%f")) - tm.assert_frame_equal(ref, res) - res = sql.read_sql_table("test_time3", self.conn) - tm.assert_frame_equal(df, res) - - def test_mixed_dtype_insert(self): - # see GH6509 - s1 = Series(2**25 + 1, dtype=np.int32) - s2 = Series(0.0, dtype=np.float32) - df = DataFrame({"s1": s1, "s2": s2}) - - # write and read again - assert df.to_sql(name="test_read_write", con=self.conn, index=False) == 1 - df2 = sql.read_sql_table("test_read_write", self.conn) - - tm.assert_frame_equal(df, df2, check_dtype=False, check_exact=True) - - def test_nan_numeric(self): - # NaNs in numeric float column - df = DataFrame({"A": [0, 1, 2], "B": [0.2, np.nan, 5.6]}) - assert df.to_sql(name="test_nan", con=self.conn, index=False) == 3 - - # with read_table - result = sql.read_sql_table("test_nan", self.conn) - tm.assert_frame_equal(result, df) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) +def test_date_parsing(conn, request): + # No Parsing + conn_name = conn + conn = request.getfixturevalue(conn) + df = sql.read_sql_table("types", conn) + expected_type = object if "sqlite" in conn_name else np.datetime64 + assert issubclass(df.DateCol.dtype.type, expected_type) - # with read_sql - result = sql.read_sql_query("SELECT * FROM test_nan", self.conn) - tm.assert_frame_equal(result, df) + df = sql.read_sql_table("types", conn, parse_dates=["DateCol"]) + assert issubclass(df.DateCol.dtype.type, np.datetime64) - def test_nan_fullcolumn(self): - # full NaN column (numeric float column) - df = DataFrame({"A": [0, 1, 2], "B": [np.nan, np.nan, np.nan]}) - assert df.to_sql(name="test_nan", con=self.conn, index=False) == 3 + df = sql.read_sql_table("types", conn, parse_dates={"DateCol": "%Y-%m-%d %H:%M:%S"}) + assert issubclass(df.DateCol.dtype.type, np.datetime64) - # with read_table - result = sql.read_sql_table("test_nan", self.conn) - tm.assert_frame_equal(result, df) + df = sql.read_sql_table( + "types", + conn, + parse_dates={"DateCol": {"format": "%Y-%m-%d %H:%M:%S"}}, + ) + assert issubclass(df.DateCol.dtype.type, np.datetime64) - # with read_sql -> not type info from table -> stays None - df["B"] = df["B"].astype("object") - df["B"] = None - result = sql.read_sql_query("SELECT * FROM test_nan", self.conn) - tm.assert_frame_equal(result, df) + df = sql.read_sql_table("types", conn, parse_dates=["IntDateCol"]) + assert issubclass(df.IntDateCol.dtype.type, np.datetime64) + + df = sql.read_sql_table("types", conn, parse_dates={"IntDateCol": "s"}) + assert issubclass(df.IntDateCol.dtype.type, np.datetime64) - def test_nan_string(self): - # NaNs in string column - df = DataFrame({"A": [0, 1, 2], "B": ["a", "b", np.nan]}) - assert df.to_sql(name="test_nan", con=self.conn, index=False) == 3 + df = sql.read_sql_table("types", conn, parse_dates={"IntDateCol": {"unit": "s"}}) + assert issubclass(df.IntDateCol.dtype.type, np.datetime64) - # NaNs are coming back as None - df.loc[2, "B"] = None - # with read_table - result = sql.read_sql_table("test_nan", self.conn) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_datetime(conn, request): + conn_name = conn + conn = request.getfixturevalue(conn) + df = DataFrame( + {"A": date_range("2013-01-01 09:00:00", periods=3), "B": np.arange(3.0)} + ) + assert df.to_sql(name="test_datetime", con=conn) == 3 + + # with read_table -> type information from schema used + result = sql.read_sql_table("test_datetime", conn) + result = result.drop("index", axis=1) + tm.assert_frame_equal(result, df) + + # with read_sql -> no type information -> sqlite has no native + result = sql.read_sql_query("SELECT * FROM test_datetime", conn) + result = result.drop("index", axis=1) + if "sqlite" in conn_name: + assert isinstance(result.loc[0, "A"], str) + result["A"] = to_datetime(result["A"]) + tm.assert_frame_equal(result, df) + else: tm.assert_frame_equal(result, df) - # with read_sql - result = sql.read_sql_query("SELECT * FROM test_nan", self.conn) + +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_datetime_NaT(conn, request): + conn_name = conn + conn = request.getfixturevalue(conn) + df = DataFrame( + {"A": date_range("2013-01-01 09:00:00", periods=3), "B": np.arange(3.0)} + ) + df.loc[1, "A"] = np.nan + assert df.to_sql(name="test_datetime", con=conn, index=False) == 3 + + # with read_table -> type information from schema used + result = sql.read_sql_table("test_datetime", conn) + tm.assert_frame_equal(result, df) + + # with read_sql -> no type information -> sqlite has no native + result = sql.read_sql_query("SELECT * FROM test_datetime", conn) + if "sqlite" in conn_name: + assert isinstance(result.loc[0, "A"], str) + result["A"] = to_datetime(result["A"], errors="coerce") + tm.assert_frame_equal(result, df) + else: tm.assert_frame_equal(result, df) - def _get_index_columns(self, tbl_name): - from sqlalchemy import inspect - insp = inspect(self.conn) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_datetime_date(conn, request): + # test support for datetime.date + conn = request.getfixturevalue(conn) + df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"]) + assert df.to_sql(name="test_date", con=conn, index=False) == 2 + res = read_sql_table("test_date", conn) + result = res["a"] + expected = to_datetime(df["a"]) + # comes back as datetime64 + tm.assert_series_equal(result, expected) - ixs = insp.get_indexes(tbl_name) - ixs = [i["column_names"] for i in ixs] - return ixs - def test_to_sql_save_index(self): - self._to_sql_save_index() +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_datetime_time(conn, request, sqlite_buildin): + # test support for datetime.time + conn_name = conn + conn = request.getfixturevalue(conn) + df = DataFrame([time(9, 0, 0), time(9, 1, 30)], columns=["a"]) + assert df.to_sql(name="test_time", con=conn, index=False) == 2 + res = read_sql_table("test_time", conn) + tm.assert_frame_equal(res, df) - def test_transactions(self): - self._transaction_test() + # GH8341 + # first, use the fallback to have the sqlite adapter put in place + sqlite_conn = sqlite_buildin + assert sql.to_sql(df, "test_time2", sqlite_conn, index=False) == 2 + res = sql.read_sql_query("SELECT * FROM test_time2", sqlite_conn) + ref = df.map(lambda _: _.strftime("%H:%M:%S.%f")) + tm.assert_frame_equal(ref, res) # check if adapter is in place + # then test if sqlalchemy is unaffected by the sqlite adapter + assert sql.to_sql(df, "test_time3", conn, index=False) == 2 + if "sqlite" in conn_name: + res = sql.read_sql_query("SELECT * FROM test_time3", conn) + ref = df.map(lambda _: _.strftime("%H:%M:%S.%f")) + tm.assert_frame_equal(ref, res) + res = sql.read_sql_table("test_time3", conn) + tm.assert_frame_equal(df, res) - def test_get_schema_create_table(self, test_frame3): - # Use a dataframe without a bool column, since MySQL converts bool to - # TINYINT (which read_sql_table returns as an int and causes a dtype - # mismatch) - from sqlalchemy import text - from sqlalchemy.engine import Engine - tbl = "test_get_schema_create_table" - create_sql = sql.get_schema(test_frame3, tbl, con=self.conn) - blank_test_df = test_frame3.iloc[:0] +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_mixed_dtype_insert(conn, request): + # see GH6509 + conn = request.getfixturevalue(conn) + s1 = Series(2**25 + 1, dtype=np.int32) + s2 = Series(0.0, dtype=np.float32) + df = DataFrame({"s1": s1, "s2": s2}) - self.drop_table(tbl, self.conn) - create_sql = text(create_sql) - if isinstance(self.conn, Engine): - with self.conn.connect() as conn: - with conn.begin(): - conn.execute(create_sql) - else: - with self.conn.begin(): - self.conn.execute(create_sql) - returned_df = sql.read_sql_table(tbl, self.conn) - tm.assert_frame_equal(returned_df, blank_test_df, check_index_type=False) - self.drop_table(tbl, self.conn) + # write and read again + assert df.to_sql(name="test_read_write", con=conn, index=False) == 1 + df2 = sql.read_sql_table("test_read_write", conn) - def test_dtype(self): - from sqlalchemy import ( - TEXT, - String, + tm.assert_frame_equal(df, df2, check_dtype=False, check_exact=True) + + +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_nan_numeric(conn, request): + # NaNs in numeric float column + conn = request.getfixturevalue(conn) + df = DataFrame({"A": [0, 1, 2], "B": [0.2, np.nan, 5.6]}) + assert df.to_sql(name="test_nan", con=conn, index=False) == 3 + + # with read_table + result = sql.read_sql_table("test_nan", conn) + tm.assert_frame_equal(result, df) + + # with read_sql + result = sql.read_sql_query("SELECT * FROM test_nan", conn) + tm.assert_frame_equal(result, df) + + +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_nan_fullcolumn(conn, request): + # full NaN column (numeric float column) + conn = request.getfixturevalue(conn) + df = DataFrame({"A": [0, 1, 2], "B": [np.nan, np.nan, np.nan]}) + assert df.to_sql(name="test_nan", con=conn, index=False) == 3 + + # with read_table + result = sql.read_sql_table("test_nan", conn) + tm.assert_frame_equal(result, df) + + # with read_sql -> not type info from table -> stays None + df["B"] = df["B"].astype("object") + df["B"] = None + result = sql.read_sql_query("SELECT * FROM test_nan", conn) + tm.assert_frame_equal(result, df) + + +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_nan_string(conn, request): + # NaNs in string column + conn = request.getfixturevalue(conn) + df = DataFrame({"A": [0, 1, 2], "B": ["a", "b", np.nan]}) + assert df.to_sql(name="test_nan", con=conn, index=False) == 3 + + # NaNs are coming back as None + df.loc[2, "B"] = None + + # with read_table + result = sql.read_sql_table("test_nan", conn) + tm.assert_frame_equal(result, df) + + # with read_sql + result = sql.read_sql_query("SELECT * FROM test_nan", conn) + tm.assert_frame_equal(result, df) + + +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +def test_to_sql_save_index(conn, request): + if "engine" in conn: + request.node.add_marker( + pytest.xfail(reason="fails and hangs forever with engine") ) - from sqlalchemy.schema import MetaData - cols = ["A", "B"] - data = [(0.8, True), (0.9, None)] - df = DataFrame(data, columns=cols) - assert df.to_sql(name="dtype_test", con=self.conn) == 2 - assert df.to_sql(name="dtype_test2", con=self.conn, dtype={"B": TEXT}) == 2 - meta = MetaData() - meta.reflect(bind=self.conn) - sqltype = meta.tables["dtype_test2"].columns["B"].type - assert isinstance(sqltype, TEXT) - msg = "The type of B is not a SQLAlchemy type" - with pytest.raises(ValueError, match=msg): - df.to_sql(name="error", con=self.conn, dtype={"B": str}) + conn_name = conn + conn = request.getfixturevalue(conn) + df = DataFrame.from_records( + [(1, 2.1, "line1"), (2, 1.5, "line2")], columns=["A", "B", "C"], index=["A"] + ) - # GH9083 - assert ( - df.to_sql(name="dtype_test3", con=self.conn, dtype={"B": String(10)}) == 2 + if conn_name == "sqlite_buildin": + pandasSQL = sql.SQLiteDatabase(conn) + tbl_name = "test_to_sql_saves_index" + assert pandasSQL.to_sql(df, tbl_name) == 2 + else: + pandasSQL = sql.SQLDatabase(conn) + tbl_name = "test_to_sql_saves_index" + assert pandasSQL.to_sql(df, tbl_name) == 2 + + if conn_name in {"sqlite_buildin", "sqlite_str"}: + ixs = sql.read_sql_query( + "SELECT * FROM sqlite_master WHERE type = 'index' " + f"AND tbl_name = '{tbl_name}'", + conn, ) - meta.reflect(bind=self.conn) - sqltype = meta.tables["dtype_test3"].columns["B"].type - assert isinstance(sqltype, String) - assert sqltype.length == 10 + ix_cols = [] + for ix_name in ixs.name: + ix_info = sql.read_sql_query(f"PRAGMA index_info({ix_name})", conn) + ix_cols.append(ix_info.name.tolist()) + else: + from sqlalchemy import inspect - # single dtype - assert df.to_sql(name="single_dtype_test", con=self.conn, dtype=TEXT) == 2 - meta.reflect(bind=self.conn) - sqltypea = meta.tables["single_dtype_test"].columns["A"].type - sqltypeb = meta.tables["single_dtype_test"].columns["B"].type - assert isinstance(sqltypea, TEXT) - assert isinstance(sqltypeb, TEXT) + insp = inspect(conn) - def test_notna_dtype(self): - from sqlalchemy import ( - Boolean, - DateTime, - Float, - Integer, + ixs = insp.get_indexes(tbl_name) + ix_cols = [i["column_names"] for i in ixs] + + assert ix_cols == [["A"]] + + +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +def test_transactions(conn, request): + conn_name = conn + conn = request.getfixturevalue(conn) + + stmt = "CREATE TABLE test_trans (A INT, B TEXT)" + if conn_name == "sqlite_buildin": + pandasSQL = sql.SQLiteDatabase(conn) + else: + pandasSQL = sql.SQLDatabase(conn) + from sqlalchemy import text + + stmt = text(stmt) + + with pandasSQL.run_transaction() as trans: + trans.execute(stmt) + + +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_get_schema_create_table(conn, request, test_frame3): + # Use a dataframe without a bool column, since MySQL converts bool to + # TINYINT (which read_sql_table returns as an int and causes a dtype + # mismatch) + if conn == "sqlite_str": + request.node.add_marker( + pytest.xfail(reason="test does not support sqlite_str fixture") ) - from sqlalchemy.schema import MetaData - cols = { - "Bool": Series([True, None]), - "Date": Series([datetime(2012, 5, 1), None]), - "Int": Series([1, None], dtype="object"), - "Float": Series([1.1, None]), + conn = request.getfixturevalue(conn) + + from sqlalchemy import text + from sqlalchemy.engine import Engine + + tbl = "test_get_schema_create_table" + create_sql = sql.get_schema(test_frame3, tbl, con=conn) + blank_test_df = test_frame3.iloc[:0] + + drop_table(tbl, conn) + create_sql = text(create_sql) + if isinstance(conn, Engine): + with conn.connect() as newcon: + with newcon.begin(): + newcon.execute(create_sql) + else: + conn.execute(create_sql) + returned_df = sql.read_sql_table(tbl, conn) + tm.assert_frame_equal(returned_df, blank_test_df, check_index_type=False) + + +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_dtype(conn, request): + if conn == "sqlite_str": + pytest.skip("sqlite_str has no inspection system") + + from sqlalchemy import ( + TEXT, + String, + ) + from sqlalchemy.schema import MetaData + + conn = request.getfixturevalue(conn) + cols = ["A", "B"] + data = [(0.8, True), (0.9, None)] + df = DataFrame(data, columns=cols) + assert df.to_sql(name="dtype_test", con=conn) == 2 + assert df.to_sql(name="dtype_test2", con=conn, dtype={"B": TEXT}) == 2 + meta = MetaData() + meta.reflect(bind=conn) + sqltype = meta.tables["dtype_test2"].columns["B"].type + assert isinstance(sqltype, TEXT) + msg = "The type of B is not a SQLAlchemy type" + with pytest.raises(ValueError, match=msg): + df.to_sql(name="error", con=conn, dtype={"B": str}) + + # GH9083 + assert df.to_sql(name="dtype_test3", con=conn, dtype={"B": String(10)}) == 2 + meta.reflect(bind=conn) + sqltype = meta.tables["dtype_test3"].columns["B"].type + assert isinstance(sqltype, String) + assert sqltype.length == 10 + + # single dtype + assert df.to_sql(name="single_dtype_test", con=conn, dtype=TEXT) == 2 + meta.reflect(bind=conn) + sqltypea = meta.tables["single_dtype_test"].columns["A"].type + sqltypeb = meta.tables["single_dtype_test"].columns["B"].type + assert isinstance(sqltypea, TEXT) + assert isinstance(sqltypeb, TEXT) + + +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_notna_dtype(conn, request): + if conn == "sqlite_str": + pytest.skip("sqlite_str has no inspection system") + + from sqlalchemy import ( + Boolean, + DateTime, + Float, + Integer, + ) + from sqlalchemy.schema import MetaData + + conn_name = conn + conn = request.getfixturevalue(conn) + + cols = { + "Bool": Series([True, None]), + "Date": Series([datetime(2012, 5, 1), None]), + "Int": Series([1, None], dtype="object"), + "Float": Series([1.1, None]), + } + df = DataFrame(cols) + + tbl = "notna_dtype_test" + assert df.to_sql(name=tbl, con=conn) == 2 + _ = sql.read_sql_table(tbl, conn) + meta = MetaData() + meta.reflect(bind=conn) + my_type = Integer if "mysql" in conn_name else Boolean + col_dict = meta.tables[tbl].columns + assert isinstance(col_dict["Bool"].type, my_type) + assert isinstance(col_dict["Date"].type, DateTime) + assert isinstance(col_dict["Int"].type, Integer) + assert isinstance(col_dict["Float"].type, Float) + + +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_double_precision(conn, request): + if conn == "sqlite_str": + pytest.skip("sqlite_str has no inspection system") + + from sqlalchemy import ( + BigInteger, + Float, + Integer, + ) + from sqlalchemy.schema import MetaData + + conn = request.getfixturevalue(conn) + V = 1.23456789101112131415 + + df = DataFrame( + { + "f32": Series([V], dtype="float32"), + "f64": Series([V], dtype="float64"), + "f64_as_f32": Series([V], dtype="float64"), + "i32": Series([5], dtype="int32"), + "i64": Series([5], dtype="int64"), } - df = DataFrame(cols) + ) - tbl = "notna_dtype_test" - assert df.to_sql(name=tbl, con=self.conn) == 2 - _ = sql.read_sql_table(tbl, self.conn) - meta = MetaData() - meta.reflect(bind=self.conn) - my_type = Integer if self.flavor == "mysql" else Boolean - col_dict = meta.tables[tbl].columns - assert isinstance(col_dict["Bool"].type, my_type) - assert isinstance(col_dict["Date"].type, DateTime) - assert isinstance(col_dict["Int"].type, Integer) - assert isinstance(col_dict["Float"].type, Float) - - def test_double_precision(self): - from sqlalchemy import ( - BigInteger, - Float, - Integer, + assert ( + df.to_sql( + name="test_dtypes", + con=conn, + index=False, + if_exists="replace", + dtype={"f64_as_f32": Float(precision=23)}, ) - from sqlalchemy.schema import MetaData + == 1 + ) + res = sql.read_sql_table("test_dtypes", conn) - V = 1.23456789101112131415 + # check precision of float64 + assert np.round(df["f64"].iloc[0], 14) == np.round(res["f64"].iloc[0], 14) - df = DataFrame( - { - "f32": Series([V], dtype="float32"), - "f64": Series([V], dtype="float64"), - "f64_as_f32": Series([V], dtype="float64"), - "i32": Series([5], dtype="int32"), - "i64": Series([5], dtype="int64"), - } - ) + # check sql types + meta = MetaData() + meta.reflect(bind=conn) + col_dict = meta.tables["test_dtypes"].columns + assert str(col_dict["f32"].type) == str(col_dict["f64_as_f32"].type) + assert isinstance(col_dict["f32"].type, Float) + assert isinstance(col_dict["f64"].type, Float) + assert isinstance(col_dict["i32"].type, Integer) + assert isinstance(col_dict["i64"].type, BigInteger) - assert ( - df.to_sql( - name="test_dtypes", - con=self.conn, - index=False, - if_exists="replace", - dtype={"f64_as_f32": Float(precision=23)}, - ) - == 1 - ) - res = sql.read_sql_table("test_dtypes", self.conn) - - # check precision of float64 - assert np.round(df["f64"].iloc[0], 14) == np.round(res["f64"].iloc[0], 14) - - # check sql types - meta = MetaData() - meta.reflect(bind=self.conn) - col_dict = meta.tables["test_dtypes"].columns - assert str(col_dict["f32"].type) == str(col_dict["f64_as_f32"].type) - assert isinstance(col_dict["f32"].type, Float) - assert isinstance(col_dict["f64"].type, Float) - assert isinstance(col_dict["i32"].type, Integer) - assert isinstance(col_dict["i64"].type, BigInteger) - - def test_connectable_issue_example(self): - # This tests the example raised in issue - # https://github.com/pandas-dev/pandas/issues/10104 - from sqlalchemy.engine import Engine - def test_select(connection): - query = "SELECT test_foo_data FROM test_foo_data" - return sql.read_sql_query(query, con=connection) +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_connectable_issue_example(conn, request): + # This tests the example raised in issue + # https://github.com/pandas-dev/pandas/issues/10104 + from sqlalchemy.engine import Engine + + def test_select(connection): + query = "SELECT test_foo_data FROM test_foo_data" + return sql.read_sql_query(query, con=connection) - def test_append(connection, data): - data.to_sql(name="test_foo_data", con=connection, if_exists="append") + def test_append(connection, data): + data.to_sql(name="test_foo_data", con=connection, if_exists="append") - def test_connectable(conn): - # https://github.com/sqlalchemy/sqlalchemy/commit/ - # 00b5c10846e800304caa86549ab9da373b42fa5d#r48323973 - foo_data = test_select(conn) - test_append(conn, foo_data) + def test_connectable(conn): + # https://github.com/sqlalchemy/sqlalchemy/commit/ + # 00b5c10846e800304caa86549ab9da373b42fa5d#r48323973 + foo_data = test_select(conn) + test_append(conn, foo_data) - def main(connectable): - if isinstance(connectable, Engine): - with connectable.connect() as conn: - with conn.begin(): - test_connectable(conn) - else: - test_connectable(connectable) + def main(connectable): + if isinstance(connectable, Engine): + with connectable.connect() as conn: + with conn.begin(): + test_connectable(conn) + else: + test_connectable(connectable) - assert ( - DataFrame({"test_foo_data": [0, 1, 2]}).to_sql( - name="test_foo_data", con=self.conn - ) - == 3 - ) - main(self.conn) + conn = request.getfixturevalue(conn) + assert ( + DataFrame({"test_foo_data": [0, 1, 2]}).to_sql(name="test_foo_data", con=conn) + == 3 + ) + main(conn) + + +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +@pytest.mark.parametrize( + "input", + [{"foo": [np.inf]}, {"foo": [-np.inf]}, {"foo": [-np.inf], "infe0": ["bar"]}], +) +def test_to_sql_with_negative_npinf(conn, request, input): + # GH 34431 + + df = DataFrame(input) + conn_name = conn + conn = request.getfixturevalue(conn) + + if "mysql" in conn_name: + # GH 36465 + # The input {"foo": [-np.inf], "infe0": ["bar"]} does not raise any error + # for pymysql version >= 0.10 + # TODO(GH#36465): remove this version check after GH 36465 is fixed + pymysql = pytest.importorskip("pymysql") + + if Version(pymysql.__version__) < Version("1.0.3") and "infe0" in df.columns: + mark = pytest.mark.xfail(reason="GH 36465") + request.node.add_marker(mark) + + msg = "inf cannot be used with MySQL" + with pytest.raises(ValueError, match=msg): + df.to_sql(name="foobar", con=conn, index=False) + else: + assert df.to_sql(name="foobar", con=conn, index=False) == 1 + res = sql.read_sql_table("foobar", conn) + tm.assert_equal(df, res) + + +@pytest.mark.db +@pytest.mark.parametrize("conn", sqlalchemy_connectable) +def test_temporary_table(conn, request): + if conn == "sqlite_str": + pytest.skip("test does not work with str connection") - @pytest.mark.parametrize( - "input", - [{"foo": [np.inf]}, {"foo": [-np.inf]}, {"foo": [-np.inf], "infe0": ["bar"]}], + from sqlalchemy import ( + Column, + Integer, + Unicode, + select, + ) + from sqlalchemy.orm import ( + Session, + declarative_base, ) - def test_to_sql_with_negative_npinf(self, input, request): - # GH 34431 - df = DataFrame(input) + conn = request.getfixturevalue(conn) + test_data = "Hello, World!" + expected = DataFrame({"spam": [test_data]}) + Base = declarative_base() + + class Temporary(Base): + __tablename__ = "temp_test" + __table_args__ = {"prefixes": ["TEMPORARY"]} + id = Column(Integer, primary_key=True) + spam = Column(Unicode(30), nullable=False) + + with Session(conn) as session: + with session.begin(): + conn = session.connection() + Temporary.__table__.create(conn) + session.add(Temporary(spam=test_data)) + session.flush() + df = sql.read_sql_query(sql=select(Temporary.spam), con=conn) + tm.assert_frame_equal(df, expected) - if self.flavor == "mysql": - # GH 36465 - # The input {"foo": [-np.inf], "infe0": ["bar"]} does not raise any error - # for pymysql version >= 0.10 - # TODO(GH#36465): remove this version check after GH 36465 is fixed - pymysql = pytest.importorskip("pymysql") - - if ( - Version(pymysql.__version__) < Version("1.0.3") - and "infe0" in df.columns - ): - mark = pytest.mark.xfail(reason="GH 36465") - request.node.add_marker(mark) - - msg = "inf cannot be used with MySQL" - with pytest.raises(ValueError, match=msg): - df.to_sql(name="foobar", con=self.conn, index=False) - else: - assert df.to_sql(name="foobar", con=self.conn, index=False) == 1 - res = sql.read_sql_table("foobar", self.conn) - tm.assert_equal(df, res) - def test_temporary_table(self): - from sqlalchemy import ( - Column, - Integer, - Unicode, - select, - ) - from sqlalchemy.orm import ( - Session, - declarative_base, +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +def test_invalid_engine(conn, request, test_frame1): + if conn == "sqlite_buildin": + request.node.add_marker( + pytest.mark.xfail(reason="SQLiteDatabase does not raise for bad engine") ) + conn = request.getfixturevalue(conn) + msg = "engine must be one of 'auto', 'sqlalchemy'" + pandasSQL = pandasSQL_builder(conn) + with pytest.raises(ValueError, match=msg): + pandasSQL.to_sql(test_frame1, "test_frame1", engine="bad_engine") - test_data = "Hello, World!" - expected = DataFrame({"spam": [test_data]}) - Base = declarative_base() - class Temporary(Base): - __tablename__ = "temp_test" - __table_args__ = {"prefixes": ["TEMPORARY"]} - id = Column(Integer, primary_key=True) - spam = Column(Unicode(30), nullable=False) - - with Session(self.conn) as session: - with session.begin(): - conn = session.connection() - Temporary.__table__.create(conn) - session.add(Temporary(spam=test_data)) - session.flush() - df = sql.read_sql_query(sql=select(Temporary.spam), con=conn) - tm.assert_frame_equal(df, expected) - - # -- SQL Engine tests (in the base class for now) - def test_invalid_engine(self, test_frame1): - msg = "engine must be one of 'auto', 'sqlalchemy'" - with pytest.raises(ValueError, match=msg): - self._to_sql_with_sql_engine(test_frame1, "bad_engine") +@pytest.mark.skip("Couldn't get this to work?") +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +def test_options_sqlalchemy(conn, request, test_frame1): + # use the set option + conn = request.getfixturevalue(conn) + with pd.option_context("io.sql.engine", "sqlalchemy"): + pandasSQL = pandasSQL_builder(conn) + assert pandasSQL.to_sql(test_frame1, "test_frame1", engine="auto") == 4 - def test_options_sqlalchemy(self, test_frame1): - # use the set option - with pd.option_context("io.sql.engine", "sqlalchemy"): - self._to_sql_with_sql_engine(test_frame1) + breakpoint() + assert pandasSQL.has_table("test_frame1") + num_entries = len(test_frame1) + num_rows = count_rows(conn, "test_frame1") + assert num_rows == num_entries - def test_options_auto(self, test_frame1): - # use the set option - with pd.option_context("io.sql.engine", "auto"): - self._to_sql_with_sql_engine(test_frame1) - def test_options_get_engine(self): +@pytest.mark.skip("Couldn't get this to work?") +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +def test_options_auto(conn, request, test_frame1): + # use the set option + with pd.option_context("io.sql.engine", "auto"): + self._to_sql_with_sql_engine(test_frame1) + + +def test_options_get_engine(): + assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine) + + with pd.option_context("io.sql.engine", "sqlalchemy"): + assert isinstance(get_engine("auto"), SQLAlchemyEngine) assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine) - with pd.option_context("io.sql.engine", "sqlalchemy"): - assert isinstance(get_engine("auto"), SQLAlchemyEngine) - assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine) + with pd.option_context("io.sql.engine", "auto"): + assert isinstance(get_engine("auto"), SQLAlchemyEngine) + assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine) - with pd.option_context("io.sql.engine", "auto"): - assert isinstance(get_engine("auto"), SQLAlchemyEngine) - assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine) - def test_get_engine_auto_error_message(self): - # Expect different error messages from get_engine(engine="auto") - # if engines aren't installed vs. are installed but bad version - pass - # TODO(GH#36893) fill this in when we add more engines +def test_get_engine_auto_error_message(): + # Expect different error messages from get_engine(engine="auto") + # if engines aren't installed vs. are installed but bad version + pass + # TODO(GH#36893) fill this in when we add more engines - @pytest.mark.parametrize("func", ["read_sql", "read_sql_query"]) - def test_read_sql_dtype_backend(self, string_storage, func, dtype_backend): - # GH#50048 - table = "test" - df = self.dtype_backend_data() - df.to_sql(name=table, con=self.conn, index=False, if_exists="replace") - with pd.option_context("mode.string_storage", string_storage): - result = getattr(pd, func)( - f"Select * from {table}", self.conn, dtype_backend=dtype_backend - ) - expected = self.dtype_backend_expected(string_storage, dtype_backend) - tm.assert_frame_equal(result, expected) +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("func", ["read_sql", "read_sql_query"]) +def test_read_sql_dtype_backend( + conn, + request, + string_storage, + func, + dtype_backend, + dtype_backend_data, + dtype_backend_expected, +): + # GH#50048 + conn_name = conn + conn = request.getfixturevalue(conn) + table = "test" + df = dtype_backend_data + df.to_sql(name=table, con=conn, index=False, if_exists="replace") - with pd.option_context("mode.string_storage", string_storage): - iterator = getattr(pd, func)( - f"Select * from {table}", - con=self.conn, - dtype_backend=dtype_backend, - chunksize=3, - ) - expected = self.dtype_backend_expected(string_storage, dtype_backend) - for result in iterator: - tm.assert_frame_equal(result, expected) + with pd.option_context("mode.string_storage", string_storage): + result = getattr(pd, func)( + f"Select * from {table}", conn, dtype_backend=dtype_backend + ) + expected = dtype_backend_expected(string_storage, dtype_backend, conn_name) + tm.assert_frame_equal(result, expected) - @pytest.mark.parametrize("func", ["read_sql", "read_sql_table"]) - def test_read_sql_dtype_backend_table(self, string_storage, func, dtype_backend): - # GH#50048 - table = "test" - df = self.dtype_backend_data() - df.to_sql(name=table, con=self.conn, index=False, if_exists="replace") + with pd.option_context("mode.string_storage", string_storage): + iterator = getattr(pd, func)( + f"Select * from {table}", + con=conn, + dtype_backend=dtype_backend, + chunksize=3, + ) + expected = dtype_backend_expected(string_storage, dtype_backend, conn_name) + for result in iterator: + tm.assert_frame_equal(result, expected) - with pd.option_context("mode.string_storage", string_storage): - result = getattr(pd, func)(table, self.conn, dtype_backend=dtype_backend) - expected = self.dtype_backend_expected(string_storage, dtype_backend) - tm.assert_frame_equal(result, expected) - with pd.option_context("mode.string_storage", string_storage): - iterator = getattr(pd, func)( - table, - self.conn, - dtype_backend=dtype_backend, - chunksize=3, +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("func", ["read_sql", "read_sql_table"]) +def test_read_sql_dtype_backend_table( + conn, + request, + string_storage, + func, + dtype_backend, + dtype_backend_data, + dtype_backend_expected, +): + if "sqlite" in conn: + request.node.add_marker( + pytest.mark.xfail( + reason=( + "SQLite actually returns proper boolean values via " + "read_sql_table, but before pytest refactor was skipped" + ) ) - expected = self.dtype_backend_expected(string_storage, dtype_backend) - for result in iterator: - tm.assert_frame_equal(result, expected) + ) + # GH#50048 + conn_name = conn + conn = request.getfixturevalue(conn) + table = "test" + df = dtype_backend_data + df.to_sql(name=table, con=conn, index=False, if_exists="replace") - @pytest.mark.parametrize("func", ["read_sql", "read_sql_table", "read_sql_query"]) - def test_read_sql_invalid_dtype_backend_table(self, func): - table = "test" - df = self.dtype_backend_data() - df.to_sql(name=table, con=self.conn, index=False, if_exists="replace") + with pd.option_context("mode.string_storage", string_storage): + result = getattr(pd, func)(table, conn, dtype_backend=dtype_backend) + expected = dtype_backend_expected(string_storage, dtype_backend, conn_name) + tm.assert_frame_equal(result, expected) - msg = ( - "dtype_backend numpy is invalid, only 'numpy_nullable' and " - "'pyarrow' are allowed." + with pd.option_context("mode.string_storage", string_storage): + iterator = getattr(pd, func)( + table, + conn, + dtype_backend=dtype_backend, + chunksize=3, ) - with pytest.raises(ValueError, match=msg): - getattr(pd, func)(table, self.conn, dtype_backend="numpy") + expected = dtype_backend_expected(string_storage, dtype_backend, conn_name) + for result in iterator: + tm.assert_frame_equal(result, expected) - def dtype_backend_data(self) -> DataFrame: - return DataFrame( - { - "a": Series([1, np.nan, 3], dtype="Int64"), - "b": Series([1, 2, 3], dtype="Int64"), - "c": Series([1.5, np.nan, 2.5], dtype="Float64"), - "d": Series([1.5, 2.0, 2.5], dtype="Float64"), - "e": [True, False, None], - "f": [True, False, True], - "g": ["a", "b", "c"], - "h": ["a", "b", None], - } - ) - def dtype_backend_expected(self, storage, dtype_backend) -> DataFrame: +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("func", ["read_sql", "read_sql_table", "read_sql_query"]) +def test_read_sql_invalid_dtype_backend_table(conn, request, func, dtype_backend_data): + conn = request.getfixturevalue(conn) + table = "test" + df = dtype_backend_data + df.to_sql(name=table, con=conn, index=False, if_exists="replace") + + msg = ( + "dtype_backend numpy is invalid, only 'numpy_nullable' and " + "'pyarrow' are allowed." + ) + with pytest.raises(ValueError, match=msg): + getattr(pd, func)(table, conn, dtype_backend="numpy") + + +@pytest.fixture +def dtype_backend_data() -> DataFrame: + return DataFrame( + { + "a": Series([1, np.nan, 3], dtype="Int64"), + "b": Series([1, 2, 3], dtype="Int64"), + "c": Series([1.5, np.nan, 2.5], dtype="Float64"), + "d": Series([1.5, 2.0, 2.5], dtype="Float64"), + "e": [True, False, None], + "f": [True, False, True], + "g": ["a", "b", "c"], + "h": ["a", "b", None], + } + ) + + +@pytest.fixture +def dtype_backend_expected(): + def func(storage, dtype_backend, conn_name): string_array: StringArray | ArrowStringArray string_array_na: StringArray | ArrowStringArray if storage == "python": @@ -3009,50 +3275,67 @@ def dtype_backend_expected(self, storage, dtype_backend) -> DataFrame: for col in df.columns } ) + + if "mysql" in conn_name or "sqlite" in conn_name: + if dtype_backend == "numpy_nullable": + df = df.astype({"e": "Int64", "f": "Int64"}) + else: + df = df.astype({"e": "int64[pyarrow]", "f": "int64[pyarrow]"}) + return df - def test_chunksize_empty_dtypes(self): - # GH#50245 - dtypes = {"a": "int64", "b": "object"} - df = DataFrame(columns=["a", "b"]).astype(dtypes) - expected = df.copy() - df.to_sql(name="test", con=self.conn, index=False, if_exists="replace") + return func - for result in read_sql_query( - "SELECT * FROM test", - self.conn, - dtype=dtypes, - chunksize=1, - ): - tm.assert_frame_equal(result, expected) - @pytest.mark.parametrize("dtype_backend", [lib.no_default, "numpy_nullable"]) - @pytest.mark.parametrize("func", ["read_sql", "read_sql_query"]) - def test_read_sql_dtype(self, func, dtype_backend): - # GH#50797 - table = "test" - df = DataFrame({"a": [1, 2, 3], "b": 5}) - df.to_sql(name=table, con=self.conn, index=False, if_exists="replace") +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +def test_chunksize_empty_dtypes(conn, request): + # GH#50245 + conn = request.getfixturevalue(conn) + dtypes = {"a": "int64", "b": "object"} + df = DataFrame(columns=["a", "b"]).astype(dtypes) + expected = df.copy() + df.to_sql(name="test", con=conn, index=False, if_exists="replace") - result = getattr(pd, func)( - f"Select * from {table}", - self.conn, - dtype={"a": np.float64}, - dtype_backend=dtype_backend, - ) - expected = DataFrame( - { - "a": Series([1, 2, 3], dtype=np.float64), - "b": Series( - [5, 5, 5], - dtype="int64" if not dtype_backend == "numpy_nullable" else "Int64", - ), - } - ) + for result in read_sql_query( + "SELECT * FROM test", + conn, + dtype=dtypes, + chunksize=1, + ): tm.assert_frame_equal(result, expected) -class TestSQLiteAlchemy(_TestSQLAlchemy): +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("dtype_backend", [lib.no_default, "numpy_nullable"]) +@pytest.mark.parametrize("func", ["read_sql", "read_sql_query"]) +def test_read_sql_dtype(conn, request, func, dtype_backend): + # GH#50797 + conn = request.getfixturevalue(conn) + table = "test" + df = DataFrame({"a": [1, 2, 3], "b": 5}) + df.to_sql(name=table, con=conn, index=False, if_exists="replace") + + result = getattr(pd, func)( + f"Select * from {table}", + conn, + dtype={"a": np.float64}, + dtype_backend=dtype_backend, + ) + expected = DataFrame( + { + "a": Series([1, 2, 3], dtype=np.float64), + "b": Series( + [5, 5, 5], + dtype="int64" if not dtype_backend == "numpy_nullable" else "Int64", + ), + } + ) + tm.assert_frame_equal(result, expected) + + +class TestSQLiteAlchemy: """ Test the sqlalchemy backend against an in-memory sqlite database. @@ -3194,7 +3477,7 @@ def test_roundtripping_datetimes(self): @pytest.mark.db -class TestMySQLAlchemy(_TestSQLAlchemy): +class TestMySQLAlchemy: """ Test the sqlalchemy backend against an MySQL database. @@ -3230,7 +3513,7 @@ def dtype_backend_expected(self, storage, dtype_backend) -> DataFrame: @pytest.mark.db -class TestPostgreSQLAlchemy(_TestSQLAlchemy): +class TestPostgreSQLAlchemy: """ Test the sqlalchemy backend against an PostgreSQL database. From e12e5d559f038211cc40c914a9c27cf899618502 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 8 Sep 2023 16:51:13 -0400 Subject: [PATCH 05/38] sqlite class conversion --- pandas/tests/io/test_sql.py | 242 +++++++++++++++++++----------------- 1 file changed, 130 insertions(+), 112 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 7a1f1089b11bc..712d0cc9757b6 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -585,6 +585,28 @@ def sqlite_buildin(): yield conn +@pytest.fixture +def sqlite_sqlalchemy_memory(iris_path, types_data): + sqlalchemy = pytest.importorskip("sqlalchemy") + engine = sqlalchemy.create_engine("sqlite:///:memory:") + + insp = sqlalchemy.inspect(engine) + if not insp.has_table("iris"): + create_and_load_iris(engine, iris_path, "sqlite") + if not insp.has_table("iris_view"): + create_and_load_iris_view(engine) + if not insp.has_table("types"): + for entry in types_data: + entry.pop("DateColWithTz") + create_and_load_types(engine, types_data, "sqlite") + + yield engine + for view in get_all_views(engine): + drop_view(view, engine) + for tbl in get_all_tables(engine): + drop_table(tbl, engine) + + @pytest.fixture def sqlite_buildin_iris(sqlite_buildin, iris_path): create_and_load_iris_sqlite3(sqlite_buildin, iris_path) @@ -620,9 +642,15 @@ def sqlite_buildin_iris(sqlite_buildin, iris_path): mysql_connectable + postgresql_connectable + sqlite_iris_connectable ) -all_connectable = sqlalchemy_connectable + ["sqlite_buildin"] +all_connectable = sqlalchemy_connectable + [ + "sqlite_buildin", + "sqlite_sqlalchemy_memory", +] -all_connectable_iris = sqlalchemy_connectable_iris + ["sqlite_buildin_iris"] +all_connectable_iris = sqlalchemy_connectable_iris + [ + "sqlite_buildin_iris", + "sqlite_sqlalchemy_memory", +] @pytest.mark.db @@ -3335,145 +3363,135 @@ def test_read_sql_dtype(conn, request, func, dtype_backend): tm.assert_frame_equal(result, expected) -class TestSQLiteAlchemy: - """ - Test the sqlalchemy backend against an in-memory sqlite database. +@pytest.mark.db +def test_keyword_deprecation(sqlite_sqlalchemy_memory): + conn = sqlite_sqlalchemy_memory + # GH 54397 + msg = ( + "Starting with pandas version 3.0 all arguments of to_sql except for the " + "arguments 'name' and 'con' will be keyword-only." + ) + df = DataFrame([{"A": 1, "B": 2, "C": 3}, {"A": 1, "B": 2, "C": 3}]) + df.to_sql("example", conn) - """ + with tm.assert_produces_warning(FutureWarning, match=msg): + df.to_sql("example", conn, None, if_exists="replace") - flavor = "sqlite" - @classmethod - def setup_engine(cls): - cls.engine = sqlalchemy.create_engine("sqlite:///:memory:") +@pytest.mark.db +def test_default_type_conversion(sqlite_sqlalchemy_memory): + conn = sqlite_sqlalchemy_memory + df = sql.read_sql_table("types", conn) - @classmethod - def setup_driver(cls): - # sqlite3 is built-in - cls.driver = None - - def test_keyword_deprecation(self): - # GH 54397 - msg = ( - "Starting with pandas version 3.0 all arguments of to_sql except for the " - "arguments 'name' and 'con' will be keyword-only." - ) - df = DataFrame([{"A": 1, "B": 2, "C": 3}, {"A": 1, "B": 2, "C": 3}]) - df.to_sql("example", self.conn) + assert issubclass(df.FloatCol.dtype.type, np.floating) + assert issubclass(df.IntCol.dtype.type, np.integer) - with tm.assert_produces_warning(FutureWarning, match=msg): - df.to_sql("example", self.conn, None, if_exists="replace") + # sqlite has no boolean type, so integer type is returned + assert issubclass(df.BoolCol.dtype.type, np.integer) - def test_default_type_conversion(self): - df = sql.read_sql_table("types", self.conn) + # Int column with NA values stays as float + assert issubclass(df.IntColWithNull.dtype.type, np.floating) - assert issubclass(df.FloatCol.dtype.type, np.floating) - assert issubclass(df.IntCol.dtype.type, np.integer) + # Non-native Bool column with NA values stays as float + assert issubclass(df.BoolColWithNull.dtype.type, np.floating) - # sqlite has no boolean type, so integer type is returned - assert issubclass(df.BoolCol.dtype.type, np.integer) - # Int column with NA values stays as float - assert issubclass(df.IntColWithNull.dtype.type, np.floating) +@pytest.mark.db +def test_default_date_load(sqlite_sqlalchemy_memory): + conn = sqlite_sqlalchemy_memory + df = sql.read_sql_table("types", conn) - # Non-native Bool column with NA values stays as float - assert issubclass(df.BoolColWithNull.dtype.type, np.floating) + # IMPORTANT - sqlite has no native date type, so shouldn't parse, but + assert not issubclass(df.DateCol.dtype.type, np.datetime64) - def test_default_date_load(self): - df = sql.read_sql_table("types", self.conn) - # IMPORTANT - sqlite has no native date type, so shouldn't parse, but - assert not issubclass(df.DateCol.dtype.type, np.datetime64) +@pytest.mark.db +def test_bigint_warning(sqlite_sqlalchemy_memory): + conn = sqlite_sqlalchemy_memory + # test no warning for BIGINT (to support int64) is raised (GH7433) + df = DataFrame({"a": [1, 2]}, dtype="int64") + assert df.to_sql(name="test_bigintwarning", con=conn, index=False) == 2 - def test_bigint_warning(self): - # test no warning for BIGINT (to support int64) is raised (GH7433) - df = DataFrame({"a": [1, 2]}, dtype="int64") - assert df.to_sql(name="test_bigintwarning", con=self.conn, index=False) == 2 + with tm.assert_produces_warning(None): + sql.read_sql_table("test_bigintwarning", conn) - with tm.assert_produces_warning(None): - sql.read_sql_table("test_bigintwarning", self.conn) - def test_valueerror_exception(self): - df = DataFrame({"col1": [1, 2], "col2": [3, 4]}) - with pytest.raises(ValueError, match="Empty table name specified"): - df.to_sql(name="", con=self.conn, if_exists="replace", index=False) +@pytest.mark.db +def test_valueerror_exception(sqlite_sqlalchemy_memory): + conn = sqlite_sqlalchemy_memory + df = DataFrame({"col1": [1, 2], "col2": [3, 4]}) + with pytest.raises(ValueError, match="Empty table name specified"): + df.to_sql(name="", con=conn, if_exists="replace", index=False) - def test_row_object_is_named_tuple(self): - # GH 40682 - # Test for the is_named_tuple() function - # Placed here due to its usage of sqlalchemy - from sqlalchemy import ( - Column, - Integer, - String, - ) - from sqlalchemy.orm import ( - declarative_base, - sessionmaker, - ) +@pytest.mark.db +def test_row_object_is_named_tuple(sqlite_sqlalchemy_memory): + conn = sqlite_sqlalchemy_memory + # GH 40682 + # Test for the is_named_tuple() function + # Placed here due to its usage of sqlalchemy - BaseModel = declarative_base() + from sqlalchemy import ( + Column, + Integer, + String, + ) + from sqlalchemy.orm import ( + declarative_base, + sessionmaker, + ) - class Test(BaseModel): - __tablename__ = "test_frame" - id = Column(Integer, primary_key=True) - string_column = Column(String(50)) + BaseModel = declarative_base() - with self.conn.begin(): - BaseModel.metadata.create_all(self.conn) - Session = sessionmaker(bind=self.conn) - with Session() as session: - df = DataFrame({"id": [0, 1], "string_column": ["hello", "world"]}) - assert ( - df.to_sql( - name="test_frame", con=self.conn, index=False, if_exists="replace" - ) - == 2 - ) - session.commit() - test_query = session.query(Test.id, Test.string_column) - df = DataFrame(test_query) + class Test(BaseModel): + __tablename__ = "test_frame" + id = Column(Integer, primary_key=True) + string_column = Column(String(50)) - assert list(df.columns) == ["id", "string_column"] + with conn.begin(): + BaseModel.metadata.create_all(conn) + Session = sessionmaker(bind=conn) + with Session() as session: + df = DataFrame({"id": [0, 1], "string_column": ["hello", "world"]}) + assert ( + df.to_sql(name="test_frame", con=conn, index=False, if_exists="replace") + == 2 + ) + session.commit() + test_query = session.query(Test.id, Test.string_column) + df = DataFrame(test_query) - def dtype_backend_expected(self, storage, dtype_backend) -> DataFrame: - df = super().dtype_backend_expected(storage, dtype_backend) - if dtype_backend == "numpy_nullable": - df = df.astype({"e": "Int64", "f": "Int64"}) - else: - df = df.astype({"e": "int64[pyarrow]", "f": "int64[pyarrow]"}) + assert list(df.columns) == ["id", "string_column"] - return df - @pytest.mark.parametrize("func", ["read_sql", "read_sql_table"]) - def test_read_sql_dtype_backend_table(self, string_storage, func): - # GH#50048 Not supported for sqlite - pass +@pytest.mark.db +def test_read_sql_string_inference(sqlite_sqlalchemy_memory): + conn = sqlite_sqlalchemy_memory + # GH#54430 + pytest.importorskip("pyarrow") + table = "test" + df = DataFrame({"a": ["x", "y"]}) + df.to_sql(table, con=conn, index=False, if_exists="replace") - def test_read_sql_string_inference(self): - # GH#54430 - pytest.importorskip("pyarrow") - table = "test" - df = DataFrame({"a": ["x", "y"]}) - df.to_sql(table, con=self.conn, index=False, if_exists="replace") + with pd.option_context("future.infer_string", True): + result = read_sql_table(table, conn) - with pd.option_context("future.infer_string", True): - result = read_sql_table(table, self.conn) + dtype = "string[pyarrow_numpy]" + expected = DataFrame( + {"a": ["x", "y"]}, dtype=dtype, columns=Index(["a"], dtype=dtype) + ) - dtype = "string[pyarrow_numpy]" - expected = DataFrame( - {"a": ["x", "y"]}, dtype=dtype, columns=Index(["a"], dtype=dtype) - ) + tm.assert_frame_equal(result, expected) - tm.assert_frame_equal(result, expected) - def test_roundtripping_datetimes(self): - # GH#54877 - df = DataFrame({"t": [datetime(2020, 12, 31, 12)]}, dtype="datetime64[ns]") - df.to_sql("test", self.conn, if_exists="replace", index=False) - result = pd.read_sql("select * from test", self.conn).iloc[0, 0] - assert result == "2020-12-31 12:00:00.000000" +@pytest.mark.db +def test_roundtripping_datetimes(sqlite_sqlalchemy_memory): + conn = sqlite_sqlalchemy_memory + # GH#54877 + df = DataFrame({"t": [datetime(2020, 12, 31, 12)]}, dtype="datetime64[ns]") + df.to_sql("test", conn, if_exists="replace", index=False) + result = pd.read_sql("select * from test", conn).iloc[0, 0] + assert result == "2020-12-31 12:00:00.000000" @pytest.mark.db From 17b8e44317458fc25d7a179189164c49ff8d055f Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Fri, 8 Sep 2023 17:16:34 -0400 Subject: [PATCH 06/38] checkpoint --- pandas/tests/io/test_sql.py | 325 +++++++++++++----------------------- 1 file changed, 117 insertions(+), 208 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 712d0cc9757b6..b1cd712f671ea 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -661,6 +661,15 @@ def test_dataframe_to_sql(conn, test_frame1, request): test_frame1.to_sql(name="test", con=conn, if_exists="append", index=False) +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +def test_dataframe_to_sql_empty(conn, test_frame1, request): + # GH 51086 if conn is sqlite_engine + conn = request.getfixturevalue(conn) + empty_df = test_frame1.iloc[:0] + empty_df.to_sql(name="test", con=conn, if_exists="append", index=False) + + @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_dataframe_to_sql_arrow_dtypes(conn, request): @@ -1168,10 +1177,6 @@ def _read_sql_iris_no_parameter_with_percent(self, sql_strings): iris_frame = self.pandasSQL.read_query(query, params=None) check_iris_frame(iris_frame) - def _to_sql_empty(self, test_frame1): - self.drop_table("test_frame1", self.conn) - assert self.pandasSQL.to_sql(test_frame1.iloc[:0], "test_frame1") == 0 - def _to_sql_with_sql_engine(self, test_frame1, engine="auto", **engine_kwargs): """`to_sql` with the `engine` param""" # mostly copied from this class's `_to_sql()` method @@ -3495,241 +3500,145 @@ def test_roundtripping_datetimes(sqlite_sqlalchemy_memory): @pytest.mark.db -class TestMySQLAlchemy: - """ - Test the sqlalchemy backend against an MySQL database. - - """ - - flavor = "mysql" - port = 3306 - - @classmethod - def setup_engine(cls): - cls.engine = sqlalchemy.create_engine( - f"mysql+{cls.driver}://root@localhost:{cls.port}/pandas", - connect_args=cls.connect_args, - ) - - @classmethod - def setup_driver(cls): - pymysql = pytest.importorskip("pymysql") - cls.driver = "pymysql" - cls.connect_args = {"client_flag": pymysql.constants.CLIENT.MULTI_STATEMENTS} - - def test_default_type_conversion(self): - pass - - def dtype_backend_expected(self, storage, dtype_backend) -> DataFrame: - df = super().dtype_backend_expected(storage, dtype_backend) - if dtype_backend == "numpy_nullable": - df = df.astype({"e": "Int64", "f": "Int64"}) - else: - df = df.astype({"e": "int64[pyarrow]", "f": "int64[pyarrow]"}) - - return df - - -@pytest.mark.db -class TestPostgreSQLAlchemy: - """ - Test the sqlalchemy backend against an PostgreSQL database. - - """ - - flavor = "postgresql" - port = 5432 - - @classmethod - def setup_engine(cls): - cls.engine = sqlalchemy.create_engine( - f"postgresql+{cls.driver}://postgres:postgres@localhost:{cls.port}/pandas" - ) - - @classmethod - def setup_driver(cls): - pytest.importorskip("psycopg2") - cls.driver = "psycopg2" - - def test_schema_support(self): - from sqlalchemy.engine import Engine +def test_psycopg2_schema_support(postgresql_psycopg2_engine): + conn = postgresql_psycopg2_engine + from sqlalchemy.engine import Engine - # only test this for postgresql (schema's not supported in - # mysql/sqlite) - df = DataFrame({"col1": [1, 2], "col2": [0.1, 0.2], "col3": ["a", "n"]}) + # only test this for postgresql (schema's not supported in + # mysql/sqlite) + df = DataFrame({"col1": [1, 2], "col2": [0.1, 0.2], "col3": ["a", "n"]}) - # create a schema - with self.conn.begin(): - self.conn.exec_driver_sql("DROP SCHEMA IF EXISTS other CASCADE;") - self.conn.exec_driver_sql("CREATE SCHEMA other;") + # create a schema + with conn.connect() as con: + with con.begin(): + con.exec_driver_sql("DROP SCHEMA IF EXISTS other CASCADE;") + con.exec_driver_sql("CREATE SCHEMA other;") - # write dataframe to different schema's - assert df.to_sql(name="test_schema_public", con=self.conn, index=False) == 2 - assert ( - df.to_sql( - name="test_schema_public_explicit", - con=self.conn, - index=False, - schema="public", - ) - == 2 + # write dataframe to different schema's + assert df.to_sql(name="test_schema_public", con=conn, index=False) == 2 + assert ( + df.to_sql( + name="test_schema_public_explicit", + con=conn, + index=False, + schema="public", ) - assert ( - df.to_sql( - name="test_schema_other", con=self.conn, index=False, schema="other" - ) - == 2 + == 2 + ) + assert ( + df.to_sql( + name="test_schema_other", con=conn, index=False, schema="other" ) + == 2 + ) - # read dataframes back in - res1 = sql.read_sql_table("test_schema_public", self.conn) - tm.assert_frame_equal(df, res1) - res2 = sql.read_sql_table("test_schema_public_explicit", self.conn) - tm.assert_frame_equal(df, res2) - res3 = sql.read_sql_table( - "test_schema_public_explicit", self.conn, schema="public" - ) - tm.assert_frame_equal(df, res3) - res4 = sql.read_sql_table("test_schema_other", self.conn, schema="other") - tm.assert_frame_equal(df, res4) - msg = "Table test_schema_other not found" - with pytest.raises(ValueError, match=msg): - sql.read_sql_table("test_schema_other", self.conn, schema="public") + # read dataframes back in + res1 = sql.read_sql_table("test_schema_public", conn) + tm.assert_frame_equal(df, res1) + res2 = sql.read_sql_table("test_schema_public_explicit", conn) + tm.assert_frame_equal(df, res2) + res3 = sql.read_sql_table( + "test_schema_public_explicit", conn, schema="public" + ) + tm.assert_frame_equal(df, res3) + res4 = sql.read_sql_table("test_schema_other", conn, schema="other") + tm.assert_frame_equal(df, res4) + msg = "Table test_schema_other not found" + with pytest.raises(ValueError, match=msg): + sql.read_sql_table("test_schema_other", conn, schema="public") - # different if_exists options + # different if_exists options - # create a schema - with self.conn.begin(): - self.conn.exec_driver_sql("DROP SCHEMA IF EXISTS other CASCADE;") - self.conn.exec_driver_sql("CREATE SCHEMA other;") + # create a schema + with conn.connect() as con: + with con.begin(): + con.exec_driver_sql("DROP SCHEMA IF EXISTS other CASCADE;") + con.exec_driver_sql("CREATE SCHEMA other;") - # write dataframe with different if_exists options - assert ( - df.to_sql( - name="test_schema_other", con=self.conn, schema="other", index=False - ) - == 2 + # write dataframe with different if_exists options + assert ( + df.to_sql( + name="test_schema_other", con=conn, schema="other", index=False ) + == 2 + ) + df.to_sql( + name="test_schema_other", + con=conn, + schema="other", + index=False, + if_exists="replace", + ) + assert ( df.to_sql( name="test_schema_other", - con=self.conn, + con=conn, schema="other", index=False, - if_exists="replace", - ) - assert ( - df.to_sql( - name="test_schema_other", - con=self.conn, - schema="other", - index=False, - if_exists="append", - ) - == 2 - ) - res = sql.read_sql_table("test_schema_other", self.conn, schema="other") - tm.assert_frame_equal(concat([df, df], ignore_index=True), res) - - # specifying schema in user-provided meta - - # The schema won't be applied on another Connection - # because of transactional schemas - if isinstance(self.conn, Engine): - engine2 = self.connect() - pdsql = sql.SQLDatabase(engine2, schema="other") - assert pdsql.to_sql(df, "test_schema_other2", index=False) == 2 - assert ( - pdsql.to_sql(df, "test_schema_other2", index=False, if_exists="replace") - == 2 - ) - assert ( - pdsql.to_sql(df, "test_schema_other2", index=False, if_exists="append") - == 2 - ) - res1 = sql.read_sql_table("test_schema_other2", self.conn, schema="other") - res2 = pdsql.read_table("test_schema_other2") - tm.assert_frame_equal(res1, res2) - - def test_self_join_date_columns(self): - # GH 44421 - from sqlalchemy.engine import Engine - from sqlalchemy.sql import text - - create_table = text( - """ - CREATE TABLE person - ( - id serial constraint person_pkey primary key, - created_dt timestamp with time zone - ); - - INSERT INTO person - VALUES (1, '2021-01-01T00:00:00Z'); - """ - ) - if isinstance(self.conn, Engine): - with self.conn.connect() as con: - with con.begin(): - con.execute(create_table) - else: - with self.conn.begin(): - self.conn.execute(create_table) - - sql_query = ( - 'SELECT * FROM "person" AS p1 INNER JOIN "person" AS p2 ON p1.id = p2.id;' + if_exists="append", ) - result = pd.read_sql(sql_query, self.conn) - expected = DataFrame( - [[1, Timestamp("2021", tz="UTC")] * 2], columns=["id", "created_dt"] * 2 - ) - tm.assert_frame_equal(result, expected) - - # Cleanup - with sql.SQLDatabase(self.conn, need_transaction=True) as pandasSQL: - pandasSQL.drop_table("person") + == 2 + ) + res = sql.read_sql_table("test_schema_other", conn, schema="other") + tm.assert_frame_equal(concat([df, df], ignore_index=True), res) -# ----------------------------------------------------------------------------- -# -- Test Sqlite / MySQL fallback +@pytest.mark.db +def test_self_join_date_columns(postgresql_psycopg2_engine): + # GH 44421 + conn = postgresql_psycopg2_engine + from sqlalchemy.engine import Engine + from sqlalchemy.sql import text -class TestSQLiteFallback: - """ - Test the fallback mode against an in-memory sqlite database. + create_table = text( + """ + CREATE TABLE person + ( + id serial constraint person_pkey primary key, + created_dt timestamp with time zone + ); + INSERT INTO person + VALUES (1, '2021-01-01T00:00:00Z'); """ + ) + with conn.connect() as con: + with con.begin(): + con.execute(create_table) - flavor = "sqlite" - - @pytest.fixture(autouse=True) - def setup_method(self, iris_path, types_data): - self.conn = self.connect() - self.load_iris_data(iris_path) - self.load_types_data(types_data) - self.pandasSQL = sql.SQLiteDatabase(self.conn) + sql_query = ( + 'SELECT * FROM "person" AS p1 INNER JOIN "person" AS p2 ON p1.id = p2.id;' + ) + result = pd.read_sql(sql_query, conn) + expected = DataFrame( + [[1, Timestamp("2021", tz="UTC")] * 2], columns=["id", "created_dt"] * 2 + ) + tm.assert_frame_equal(result, expected) - def test_read_sql_parameter(self, sql_strings): - self._read_sql_iris_parameter(sql_strings) + # Cleanup + with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: + pandasSQL.drop_table("person") - def test_read_sql_named_parameter(self, sql_strings): - self._read_sql_iris_named_parameter(sql_strings) - def test_to_sql_empty(self, test_frame1): - self._to_sql_empty(test_frame1) +@pytest.mark.db +def test_create_and_drop_table(sqlite_sqlalchemy_memory): + conn = sqlite_sqlalchemy_memory + temp_frame = DataFrame( + {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} + ) + pandasSQL = sql.SQLiteDatabase(conn) - def test_create_and_drop_table(self): - temp_frame = DataFrame( - {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} - ) + assert pandasSQL.to_sql(temp_frame, "drop_test_frame") == 4 - assert self.pandasSQL.to_sql(temp_frame, "drop_test_frame") == 4 + assert pandasSQL.has_table("drop_test_frame") - assert self.pandasSQL.has_table("drop_test_frame") + pandasSQL.drop_table("drop_test_frame") - self.pandasSQL.drop_table("drop_test_frame") + assert not pandasSQL.has_table("drop_test_frame") - assert not self.pandasSQL.has_table("drop_test_frame") +class Foo: def test_roundtrip(self, test_frame1): self._roundtrip(test_frame1) From 0a644f589b513535d2251f77a181db8566f15ad9 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Sep 2023 10:06:04 -0400 Subject: [PATCH 07/38] sqlitefallback conversion --- pandas/tests/io/test_sql.py | 249 ++++++++++++++++-------------------- 1 file changed, 108 insertions(+), 141 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index b1cd712f671ea..f3f1fabac062b 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -3502,7 +3502,6 @@ def test_roundtripping_datetimes(sqlite_sqlalchemy_memory): @pytest.mark.db def test_psycopg2_schema_support(postgresql_psycopg2_engine): conn = postgresql_psycopg2_engine - from sqlalchemy.engine import Engine # only test this for postgresql (schema's not supported in # mysql/sqlite) @@ -3526,10 +3525,7 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): == 2 ) assert ( - df.to_sql( - name="test_schema_other", con=conn, index=False, schema="other" - ) - == 2 + df.to_sql(name="test_schema_other", con=conn, index=False, schema="other") == 2 ) # read dataframes back in @@ -3537,9 +3533,7 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): tm.assert_frame_equal(df, res1) res2 = sql.read_sql_table("test_schema_public_explicit", conn) tm.assert_frame_equal(df, res2) - res3 = sql.read_sql_table( - "test_schema_public_explicit", conn, schema="public" - ) + res3 = sql.read_sql_table("test_schema_public_explicit", conn, schema="public") tm.assert_frame_equal(df, res3) res4 = sql.read_sql_table("test_schema_other", conn, schema="other") tm.assert_frame_equal(df, res4) @@ -3557,10 +3551,7 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): # write dataframe with different if_exists options assert ( - df.to_sql( - name="test_schema_other", con=conn, schema="other", index=False - ) - == 2 + df.to_sql(name="test_schema_other", con=conn, schema="other", index=False) == 2 ) df.to_sql( name="test_schema_other", @@ -3583,12 +3574,10 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): tm.assert_frame_equal(concat([df, df], ignore_index=True), res) - @pytest.mark.db def test_self_join_date_columns(postgresql_psycopg2_engine): # GH 44421 conn = postgresql_psycopg2_engine - from sqlalchemy.engine import Engine from sqlalchemy.sql import text create_table = text( @@ -3624,9 +3613,7 @@ def test_self_join_date_columns(postgresql_psycopg2_engine): @pytest.mark.db def test_create_and_drop_table(sqlite_sqlalchemy_memory): conn = sqlite_sqlalchemy_memory - temp_frame = DataFrame( - {"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]} - ) + temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) pandasSQL = sql.SQLiteDatabase(conn) assert pandasSQL.to_sql(temp_frame, "drop_test_frame") == 4 @@ -3638,138 +3625,118 @@ def test_create_and_drop_table(sqlite_sqlalchemy_memory): assert not pandasSQL.has_table("drop_test_frame") -class Foo: - def test_roundtrip(self, test_frame1): - self._roundtrip(test_frame1) - - def test_execute_sql(self): - self._execute_sql() - - def test_datetime_date(self): - # test support for datetime.date - df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"]) - assert df.to_sql(name="test_date", con=self.conn, index=False) == 2 - res = read_sql_query("SELECT * FROM test_date", self.conn) - if self.flavor == "sqlite": - # comes back as strings - tm.assert_frame_equal(res, df.astype(str)) - elif self.flavor == "mysql": - tm.assert_frame_equal(res, df) - - @pytest.mark.parametrize("tz_aware", [False, True]) - def test_datetime_time(self, tz_aware): - # test support for datetime.time, GH #8341 - if not tz_aware: - tz_times = [time(9, 0, 0), time(9, 1, 30)] - else: - tz_dt = date_range("2013-01-01 09:00:00", periods=2, tz="US/Pacific") - tz_times = Series(tz_dt.to_pydatetime()).map(lambda dt: dt.timetz()) +@pytest.mark.db +def test_sqlite_datetime_date(sqlite_buildin): + conn = sqlite_buildin + df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"]) + assert df.to_sql(name="test_date", con=conn, index=False) == 2 + res = read_sql_query("SELECT * FROM test_date", conn) + # comes back as strings + tm.assert_frame_equal(res, df.astype(str)) - df = DataFrame(tz_times, columns=["a"]) - assert df.to_sql(name="test_time", con=self.conn, index=False) == 2 - res = read_sql_query("SELECT * FROM test_time", self.conn) - if self.flavor == "sqlite": - # comes back as strings - expected = df.map(lambda _: _.strftime("%H:%M:%S.%f")) - tm.assert_frame_equal(res, expected) +@pytest.mark.db +@pytest.mark.parametrize("tz_aware", [False, True]) +def test_sqlite_datetime_time(tz_aware, sqlite_buildin): + conn = sqlite_buildin + # test support for datetime.time, GH #8341 + if not tz_aware: + tz_times = [time(9, 0, 0), time(9, 1, 30)] + else: + tz_dt = date_range("2013-01-01 09:00:00", periods=2, tz="US/Pacific") + tz_times = Series(tz_dt.to_pydatetime()).map(lambda dt: dt.timetz()) - def _get_index_columns(self, tbl_name): - ixs = sql.read_sql_query( - "SELECT * FROM sqlite_master WHERE type = 'index' " - f"AND tbl_name = '{tbl_name}'", - self.conn, - ) - ix_cols = [] - for ix_name in ixs.name: - ix_info = sql.read_sql_query(f"PRAGMA index_info({ix_name})", self.conn) - ix_cols.append(ix_info.name.tolist()) - return ix_cols - - def test_to_sql_save_index(self): - self._to_sql_save_index() - - def test_transactions(self): - self._transaction_test() - - def _get_sqlite_column_type(self, table, column): - recs = self.conn.execute(f"PRAGMA table_info({table})") - for cid, name, ctype, not_null, default, pk in recs: - if name == column: - return ctype - raise ValueError(f"Table {table}, column {column} not found") - - def test_dtype(self): - if self.flavor == "mysql": - pytest.skip("Not applicable to MySQL legacy") - cols = ["A", "B"] - data = [(0.8, True), (0.9, None)] - df = DataFrame(data, columns=cols) - assert df.to_sql(name="dtype_test", con=self.conn) == 2 - assert df.to_sql(name="dtype_test2", con=self.conn, dtype={"B": "STRING"}) == 2 - - # sqlite stores Boolean values as INTEGER - assert self._get_sqlite_column_type("dtype_test", "B") == "INTEGER" - - assert self._get_sqlite_column_type("dtype_test2", "B") == "STRING" - msg = r"B \(\) not a string" - with pytest.raises(ValueError, match=msg): - df.to_sql(name="error", con=self.conn, dtype={"B": bool}) - - # single dtype - assert df.to_sql(name="single_dtype_test", con=self.conn, dtype="STRING") == 2 - assert self._get_sqlite_column_type("single_dtype_test", "A") == "STRING" - assert self._get_sqlite_column_type("single_dtype_test", "B") == "STRING" - - def test_notna_dtype(self): - if self.flavor == "mysql": - pytest.skip("Not applicable to MySQL legacy") - - cols = { - "Bool": Series([True, None]), - "Date": Series([datetime(2012, 5, 1), None]), - "Int": Series([1, None], dtype="object"), - "Float": Series([1.1, None]), - } - df = DataFrame(cols) + df = DataFrame(tz_times, columns=["a"]) + + assert df.to_sql(name="test_time", con=conn, index=False) == 2 + res = read_sql_query("SELECT * FROM test_time", conn) + # comes back as strings + expected = df.map(lambda _: _.strftime("%H:%M:%S.%f")) + tm.assert_frame_equal(res, expected) - tbl = "notna_dtype_test" - assert df.to_sql(name=tbl, con=self.conn) == 2 - assert self._get_sqlite_column_type(tbl, "Bool") == "INTEGER" - assert self._get_sqlite_column_type(tbl, "Date") == "TIMESTAMP" - assert self._get_sqlite_column_type(tbl, "Int") == "INTEGER" - assert self._get_sqlite_column_type(tbl, "Float") == "REAL" +def get_sqlite_column_type(conn, table, column): + recs = conn.execute(f"PRAGMA table_info({table})") + for cid, name, ctype, not_null, default, pk in recs: + if name == column: + return ctype + raise ValueError(f"Table {table}, column {column} not found") - def test_illegal_names(self): - # For sqlite, these should work fine - df = DataFrame([[1, 2], [3, 4]], columns=["a", "b"]) - msg = "Empty table or column name specified" - with pytest.raises(ValueError, match=msg): - df.to_sql(name="", con=self.conn) - - for ndx, weird_name in enumerate( - [ - "test_weird_name]", - "test_weird_name[", - "test_weird_name`", - 'test_weird_name"', - "test_weird_name'", - "_b.test_weird_name_01-30", - '"_b.test_weird_name_01-30"', - "99beginswithnumber", - "12345", - "\xe9", - ] - ): - assert df.to_sql(name=weird_name, con=self.conn) == 2 - sql.table_exists(weird_name, self.conn) +@pytest.mark.db +def test_sqlite_test_dtype(sqlite_buildin): + conn = sqlite_buildin + cols = ["A", "B"] + data = [(0.8, True), (0.9, None)] + df = DataFrame(data, columns=cols) + assert df.to_sql(name="dtype_test", con=conn) == 2 + assert df.to_sql(name="dtype_test2", con=conn, dtype={"B": "STRING"}) == 2 + + # sqlite stores Boolean values as INTEGER + assert get_sqlite_column_type(conn, "dtype_test", "B") == "INTEGER" + + assert get_sqlite_column_type(conn, "dtype_test2", "B") == "STRING" + msg = r"B \(\) not a string" + with pytest.raises(ValueError, match=msg): + df.to_sql(name="error", con=conn, dtype={"B": bool}) + + # single dtype + assert df.to_sql(name="single_dtype_test", con=conn, dtype="STRING") == 2 + assert get_sqlite_column_type(conn, "single_dtype_test", "A") == "STRING" + assert get_sqlite_column_type(conn, "single_dtype_test", "B") == "STRING" + + +@pytest.mark.db +def test_sqlite_notna_dtype(sqlite_buildin): + conn = sqlite_buildin + cols = { + "Bool": Series([True, None]), + "Date": Series([datetime(2012, 5, 1), None]), + "Int": Series([1, None], dtype="object"), + "Float": Series([1.1, None]), + } + df = DataFrame(cols) + + tbl = "notna_dtype_test" + assert df.to_sql(name=tbl, con=conn) == 2 + + assert get_sqlite_column_type(conn, tbl, "Bool") == "INTEGER" + assert get_sqlite_column_type(conn, tbl, "Date") == "TIMESTAMP" + assert get_sqlite_column_type(conn, tbl, "Int") == "INTEGER" + assert get_sqlite_column_type(conn, tbl, "Float") == "REAL" + + +@pytest.mark.db +def test_sqlite_illegal_names(sqlite_buildin): + # For sqlite, these should work fine + conn = sqlite_buildin + df = DataFrame([[1, 2], [3, 4]], columns=["a", "b"]) + + msg = "Empty table or column name specified" + with pytest.raises(ValueError, match=msg): + df.to_sql(name="", con=conn) + + for ndx, weird_name in enumerate( + [ + "test_weird_name]", + "test_weird_name[", + "test_weird_name`", + 'test_weird_name"', + "test_weird_name'", + "_b.test_weird_name_01-30", + '"_b.test_weird_name_01-30"', + "99beginswithnumber", + "12345", + "\xe9", + ] + ): + assert df.to_sql(name=weird_name, con=conn) == 2 + sql.table_exists(weird_name, conn) - df2 = DataFrame([[1, 2], [3, 4]], columns=["a", weird_name]) - c_tbl = f"test_weird_col_name{ndx:d}" - assert df2.to_sql(name=c_tbl, con=self.conn) == 2 - sql.table_exists(c_tbl, self.conn) + df2 = DataFrame([[1, 2], [3, 4]], columns=["a", weird_name]) + c_tbl = f"test_weird_col_name{ndx:d}" + assert df2.to_sql(name=c_tbl, con=conn) == 2 + sql.table_exists(c_tbl, conn) # ----------------------------------------------------------------------------- From 7efc9f5c479750c4c8364deec33a61bc923c63ff Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Sep 2023 11:49:52 -0400 Subject: [PATCH 08/38] fixup tests --- pandas/tests/io/test_sql.py | 404 ++++++++++++++++++------------------ 1 file changed, 205 insertions(+), 199 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index f3f1fabac062b..bea5ebadeb4b9 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1615,7 +1615,9 @@ def test_api_to_sql_index_label_multiindex(conn, request): conn_name = conn if "mysql" in conn_name: request.node.add_marker( - pytest.mark.xfail(reason="MySQL can fail using TEXT without length as key") + pytest.mark.xfail( + reason="MySQL can fail using TEXT without length as key", strict=False + ) ) conn = request.getfixturevalue(conn) @@ -3614,7 +3616,7 @@ def test_self_join_date_columns(postgresql_psycopg2_engine): def test_create_and_drop_table(sqlite_sqlalchemy_memory): conn = sqlite_sqlalchemy_memory temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) - pandasSQL = sql.SQLiteDatabase(conn) + pandasSQL = sql.SQLDatabase(conn) assert pandasSQL.to_sql(temp_frame, "drop_test_frame") == 4 @@ -3775,227 +3777,231 @@ def tquery(query, con=None): return None if res is None else list(res) -class TestXSQLite: - def drop_table(self, table_name, conn): - cur = conn.cursor() - cur.execute(f"DROP TABLE IF EXISTS {sql._get_valid_sqlite_name(table_name)}") - conn.commit() +@pytest.mark.db +def test_xsqlite_basic(sqlite_buildin): + frame = tm.makeTimeDataFrame() + assert sql.to_sql(frame, name="test_table", con=sqlite_buildin, index=False) == 30 + result = sql.read_sql("select * from test_table", sqlite_buildin) - def test_basic(self, sqlite_buildin): - frame = tm.makeTimeDataFrame() - assert ( - sql.to_sql(frame, name="test_table", con=sqlite_buildin, index=False) == 30 - ) - result = sql.read_sql("select * from test_table", sqlite_buildin) + # HACK! Change this once indexes are handled properly. + result.index = frame.index - # HACK! Change this once indexes are handled properly. - result.index = frame.index + expected = frame + tm.assert_frame_equal(result, frame) - expected = frame - tm.assert_frame_equal(result, frame) + frame["txt"] = ["a"] * len(frame) + frame2 = frame.copy() + new_idx = Index(np.arange(len(frame2)), dtype=np.int64) + 10 + frame2["Idx"] = new_idx.copy() + assert sql.to_sql(frame2, name="test_table2", con=sqlite_buildin, index=False) == 30 + result = sql.read_sql("select * from test_table2", sqlite_buildin, index_col="Idx") + expected = frame.copy() + expected.index = new_idx + expected.index.name = "Idx" + tm.assert_frame_equal(expected, result) - frame["txt"] = ["a"] * len(frame) - frame2 = frame.copy() - new_idx = Index(np.arange(len(frame2)), dtype=np.int64) + 10 - frame2["Idx"] = new_idx.copy() - assert ( - sql.to_sql(frame2, name="test_table2", con=sqlite_buildin, index=False) - == 30 - ) - result = sql.read_sql( - "select * from test_table2", sqlite_buildin, index_col="Idx" - ) - expected = frame.copy() - expected.index = new_idx - expected.index.name = "Idx" - tm.assert_frame_equal(expected, result) - - def test_write_row_by_row(self, sqlite_buildin): - frame = tm.makeTimeDataFrame() - frame.iloc[0, 0] = np.nan - create_sql = sql.get_schema(frame, "test") - cur = sqlite_buildin.cursor() - cur.execute(create_sql) - ins = "INSERT INTO test VALUES (%s, %s, %s, %s)" - for _, row in frame.iterrows(): - fmt_sql = format_query(ins, *row) - tquery(fmt_sql, con=sqlite_buildin) +@pytest.mark.db +def test_xsqlite_write_row_by_row(sqlite_buildin): + frame = tm.makeTimeDataFrame() + frame.iloc[0, 0] = np.nan + create_sql = sql.get_schema(frame, "test") + cur = sqlite_buildin.cursor() + cur.execute(create_sql) + + ins = "INSERT INTO test VALUES (%s, %s, %s, %s)" + for _, row in frame.iterrows(): + fmt_sql = format_query(ins, *row) + tquery(fmt_sql, con=sqlite_buildin) - sqlite_buildin.commit() + sqlite_buildin.commit() - result = sql.read_sql("select * from test", con=sqlite_buildin) - result.index = frame.index - tm.assert_frame_equal(result, frame, rtol=1e-3) + result = sql.read_sql("select * from test", con=sqlite_buildin) + result.index = frame.index + tm.assert_frame_equal(result, frame, rtol=1e-3) - def test_execute(self, sqlite_buildin): - frame = tm.makeTimeDataFrame() - create_sql = sql.get_schema(frame, "test") - cur = sqlite_buildin.cursor() - cur.execute(create_sql) - ins = "INSERT INTO test VALUES (?, ?, ?, ?)" - - row = frame.iloc[0] - with sql.pandasSQL_builder(sqlite_buildin) as pandas_sql: - pandas_sql.execute(ins, tuple(row)) - sqlite_buildin.commit() - - result = sql.read_sql("select * from test", sqlite_buildin) - result.index = frame.index[:1] - tm.assert_frame_equal(result, frame[:1]) - - def test_schema(self, sqlite_buildin): - frame = tm.makeTimeDataFrame() - create_sql = sql.get_schema(frame, "test") - lines = create_sql.splitlines() - for line in lines: - tokens = line.split(" ") - if len(tokens) == 2 and tokens[0] == "A": - assert tokens[1] == "DATETIME" - - create_sql = sql.get_schema(frame, "test", keys=["A", "B"]) - lines = create_sql.splitlines() - assert 'PRIMARY KEY ("A", "B")' in create_sql - cur = sqlite_buildin.cursor() - cur.execute(create_sql) - def test_execute_fail(self, sqlite_buildin): - create_sql = """ - CREATE TABLE test - ( - a TEXT, - b TEXT, - c REAL, - PRIMARY KEY (a, b) - ); - """ - cur = sqlite_buildin.cursor() +@pytest.mark.db +def test_xsqlite_execute(sqlite_buildin): + frame = tm.makeTimeDataFrame() + create_sql = sql.get_schema(frame, "test") + cur = sqlite_buildin.cursor() + cur.execute(create_sql) + ins = "INSERT INTO test VALUES (?, ?, ?, ?)" + + row = frame.iloc[0] + with sql.pandasSQL_builder(sqlite_buildin) as pandas_sql: + pandas_sql.execute(ins, tuple(row)) + sqlite_buildin.commit() + + result = sql.read_sql("select * from test", sqlite_buildin) + result.index = frame.index[:1] + tm.assert_frame_equal(result, frame[:1]) + + +@pytest.mark.db +def test_xsqlite_schema(sqlite_buildin): + frame = tm.makeTimeDataFrame() + create_sql = sql.get_schema(frame, "test") + lines = create_sql.splitlines() + for line in lines: + tokens = line.split(" ") + if len(tokens) == 2 and tokens[0] == "A": + assert tokens[1] == "DATETIME" + + create_sql = sql.get_schema(frame, "test", keys=["A", "B"]) + lines = create_sql.splitlines() + assert 'PRIMARY KEY ("A", "B")' in create_sql + cur = sqlite_buildin.cursor() + cur.execute(create_sql) + + +@pytest.mark.db +def test_xsqlite_execute_fail(sqlite_buildin): + create_sql = """ + CREATE TABLE test + ( + a TEXT, + b TEXT, + c REAL, + PRIMARY KEY (a, b) + ); + """ + cur = sqlite_buildin.cursor() + cur.execute(create_sql) + + with sql.pandasSQL_builder(sqlite_buildin) as pandas_sql: + pandas_sql.execute('INSERT INTO test VALUES("foo", "bar", 1.234)') + pandas_sql.execute('INSERT INTO test VALUES("foo", "baz", 2.567)') + + with pytest.raises(sql.DatabaseError, match="Execution failed on sql"): + pandas_sql.execute('INSERT INTO test VALUES("foo", "bar", 7)') + + +@pytest.mark.db +def test_xsqlite_execute_closed_connection(): + create_sql = """ + CREATE TABLE test + ( + a TEXT, + b TEXT, + c REAL, + PRIMARY KEY (a, b) + ); + """ + with contextlib.closing(sqlite3.connect(":memory:")) as conn: + cur = conn.cursor() cur.execute(create_sql) - with sql.pandasSQL_builder(sqlite_buildin) as pandas_sql: + with sql.pandasSQL_builder(conn) as pandas_sql: pandas_sql.execute('INSERT INTO test VALUES("foo", "bar", 1.234)') - pandas_sql.execute('INSERT INTO test VALUES("foo", "baz", 2.567)') - with pytest.raises(sql.DatabaseError, match="Execution failed on sql"): - pandas_sql.execute('INSERT INTO test VALUES("foo", "bar", 7)') + msg = "Cannot operate on a closed database." + with pytest.raises(sqlite3.ProgrammingError, match=msg): + tquery("select * from test", con=conn) - def test_execute_closed_connection(self): - create_sql = """ - CREATE TABLE test - ( - a TEXT, - b TEXT, - c REAL, - PRIMARY KEY (a, b) - ); - """ - with contextlib.closing(sqlite3.connect(":memory:")) as conn: - cur = conn.cursor() - cur.execute(create_sql) - - with sql.pandasSQL_builder(conn) as pandas_sql: - pandas_sql.execute('INSERT INTO test VALUES("foo", "bar", 1.234)') - - msg = "Cannot operate on a closed database." - with pytest.raises(sqlite3.ProgrammingError, match=msg): - tquery("select * from test", con=conn) - - def test_keyword_as_column_names(self, sqlite_buildin): - df = DataFrame({"From": np.ones(5)}) - assert sql.to_sql(df, con=sqlite_buildin, name="testkeywords", index=False) == 5 - - def test_onecolumn_of_integer(self, sqlite_buildin): - # GH 3628 - # a column_of_integers dataframe should transfer well to sql - - mono_df = DataFrame([1, 2], columns=["c0"]) - assert sql.to_sql(mono_df, con=sqlite_buildin, name="mono_df", index=False) == 2 - # computing the sum via sql - con_x = sqlite_buildin - the_sum = sum(my_c0[0] for my_c0 in con_x.execute("select * from mono_df")) - # it should not fail, and gives 3 ( Issue #3628 ) - assert the_sum == 3 - - result = sql.read_sql("select * from mono_df", con_x) - tm.assert_frame_equal(result, mono_df) - - def test_if_exists(self, sqlite_buildin): - df_if_exists_1 = DataFrame({"col1": [1, 2], "col2": ["A", "B"]}) - df_if_exists_2 = DataFrame({"col1": [3, 4, 5], "col2": ["C", "D", "E"]}) - table_name = "table_if_exists" - sql_select = f"SELECT * FROM {table_name}" - - msg = "'notvalidvalue' is not valid for if_exists" - with pytest.raises(ValueError, match=msg): - sql.to_sql( - frame=df_if_exists_1, - con=sqlite_buildin, - name=table_name, - if_exists="notvalidvalue", - ) - self.drop_table(table_name, sqlite_buildin) - # test if_exists='fail' +@pytest.mark.db +def test_xsqlite_keyword_as_column_names(sqlite_buildin): + df = DataFrame({"From": np.ones(5)}) + assert sql.to_sql(df, con=sqlite_buildin, name="testkeywords", index=False) == 5 + + +@pytest.mark.db +def test_xsqlite_onecolumn_of_integer(sqlite_buildin): + # GH 3628 + # a column_of_integers dataframe should transfer well to sql + + mono_df = DataFrame([1, 2], columns=["c0"]) + assert sql.to_sql(mono_df, con=sqlite_buildin, name="mono_df", index=False) == 2 + # computing the sum via sql + con_x = sqlite_buildin + the_sum = sum(my_c0[0] for my_c0 in con_x.execute("select * from mono_df")) + # it should not fail, and gives 3 ( Issue #3628 ) + assert the_sum == 3 + + result = sql.read_sql("select * from mono_df", con_x) + tm.assert_frame_equal(result, mono_df) + + +@pytest.mark.db +def test_xsqlite_if_exists(sqlite_buildin): + df_if_exists_1 = DataFrame({"col1": [1, 2], "col2": ["A", "B"]}) + df_if_exists_2 = DataFrame({"col1": [3, 4, 5], "col2": ["C", "D", "E"]}) + table_name = "table_if_exists" + sql_select = f"SELECT * FROM {table_name}" + + msg = "'notvalidvalue' is not valid for if_exists" + with pytest.raises(ValueError, match=msg): sql.to_sql( - frame=df_if_exists_1, con=sqlite_buildin, name=table_name, if_exists="fail" + frame=df_if_exists_1, + con=sqlite_buildin, + name=table_name, + if_exists="notvalidvalue", ) - msg = "Table 'table_if_exists' already exists" - with pytest.raises(ValueError, match=msg): - sql.to_sql( - frame=df_if_exists_1, - con=sqlite_buildin, - name=table_name, - if_exists="fail", - ) - # test if_exists='replace' + drop_table(table_name, sqlite_buildin) + + # test if_exists='fail' + sql.to_sql( + frame=df_if_exists_1, con=sqlite_buildin, name=table_name, if_exists="fail" + ) + msg = "Table 'table_if_exists' already exists" + with pytest.raises(ValueError, match=msg): sql.to_sql( frame=df_if_exists_1, con=sqlite_buildin, name=table_name, + if_exists="fail", + ) + # test if_exists='replace' + sql.to_sql( + frame=df_if_exists_1, + con=sqlite_buildin, + name=table_name, + if_exists="replace", + index=False, + ) + assert tquery(sql_select, con=sqlite_buildin) == [(1, "A"), (2, "B")] + assert ( + sql.to_sql( + frame=df_if_exists_2, + con=sqlite_buildin, + name=table_name, if_exists="replace", index=False, ) - assert tquery(sql_select, con=sqlite_buildin) == [(1, "A"), (2, "B")] - assert ( - sql.to_sql( - frame=df_if_exists_2, - con=sqlite_buildin, - name=table_name, - if_exists="replace", - index=False, - ) - == 3 - ) - assert tquery(sql_select, con=sqlite_buildin) == [(3, "C"), (4, "D"), (5, "E")] - self.drop_table(table_name, sqlite_buildin) + == 3 + ) + assert tquery(sql_select, con=sqlite_buildin) == [(3, "C"), (4, "D"), (5, "E")] + drop_table(table_name, sqlite_buildin) - # test if_exists='append' - assert ( - sql.to_sql( - frame=df_if_exists_1, - con=sqlite_buildin, - name=table_name, - if_exists="fail", - index=False, - ) - == 2 + # test if_exists='append' + assert ( + sql.to_sql( + frame=df_if_exists_1, + con=sqlite_buildin, + name=table_name, + if_exists="fail", + index=False, ) - assert tquery(sql_select, con=sqlite_buildin) == [(1, "A"), (2, "B")] - assert ( - sql.to_sql( - frame=df_if_exists_2, - con=sqlite_buildin, - name=table_name, - if_exists="append", - index=False, - ) - == 3 + == 2 + ) + assert tquery(sql_select, con=sqlite_buildin) == [(1, "A"), (2, "B")] + assert ( + sql.to_sql( + frame=df_if_exists_2, + con=sqlite_buildin, + name=table_name, + if_exists="append", + index=False, ) - assert tquery(sql_select, con=sqlite_buildin) == [ - (1, "A"), - (2, "B"), - (3, "C"), - (4, "D"), - (5, "E"), - ] - self.drop_table(table_name, sqlite_buildin) + == 3 + ) + assert tquery(sql_select, con=sqlite_buildin) == [ + (1, "A"), + (2, "B"), + (3, "C"), + (4, "D"), + (5, "E"), + ] + drop_table(table_name, sqlite_buildin) From d1b2dbd6df38e8395ace20da3b1c05b637675011 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Sep 2023 12:10:29 -0400 Subject: [PATCH 09/38] no more test classes --- pandas/tests/io/test_sql.py | 215 +++++++++++++++++------------------- 1 file changed, 104 insertions(+), 111 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index bea5ebadeb4b9..34dc7e98f7d38 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1136,128 +1136,64 @@ def test_execute_deprecated(sqlite_buildin_iris): sql.execute("select * from iris", sqlite_buildin_iris) -class PandasSQLTest: - """ - Base class with common private methods for SQLAlchemy and fallback cases. - - """ - - def load_iris_data(self, iris_path): - self.drop_view("iris_view", self.conn) - self.drop_table("iris", self.conn) - if isinstance(self.conn, sqlite3.Connection): - create_and_load_iris_sqlite3(self.conn, iris_path) - else: - create_and_load_iris(self.conn, iris_path, self.flavor) - - def load_types_data(self, types_data): - if self.flavor != "postgresql": - for entry in types_data: - entry.pop("DateColWithTz") - if isinstance(self.conn, sqlite3.Connection): - types_data = [tuple(entry.values()) for entry in types_data] - create_and_load_types_sqlite3(self.conn, types_data) - else: - create_and_load_types(self.conn, types_data, self.flavor) - - def _read_sql_iris_parameter(self, sql_strings): - query = sql_strings["read_parameters"][self.flavor] - params = ("Iris-setosa", 5.1) - iris_frame = self.pandasSQL.read_query(query, params=params) - check_iris_frame(iris_frame) - - def _read_sql_iris_named_parameter(self, sql_strings): - query = sql_strings["read_named_parameters"][self.flavor] - params = {"name": "Iris-setosa", "length": 5.1} - iris_frame = self.pandasSQL.read_query(query, params=params) - check_iris_frame(iris_frame) - - def _read_sql_iris_no_parameter_with_percent(self, sql_strings): - query = sql_strings["read_no_parameters_with_percent"][self.flavor] - iris_frame = self.pandasSQL.read_query(query, params=None) - check_iris_frame(iris_frame) - - def _to_sql_with_sql_engine(self, test_frame1, engine="auto", **engine_kwargs): - """`to_sql` with the `engine` param""" - # mostly copied from this class's `_to_sql()` method - self.drop_table("test_frame1", self.conn) - - assert ( - self.pandasSQL.to_sql( - test_frame1, "test_frame1", engine=engine, **engine_kwargs - ) - == 4 - ) - assert self.pandasSQL.has_table("test_frame1") - - num_entries = len(test_frame1) - num_rows = count_rows(self.conn, "test_frame1") - assert num_rows == num_entries +@pytest.fixture +def flavor(): + def func(conn_name): + if "postgresql" in conn_name: + return "postgresql" + elif "sqlite" in conn_name: + return "sqlite" + elif "mysql" in conn_name: + return "mysql" - # Nuke table - self.drop_table("test_frame1", self.conn) + raise ValueError(f"unsupported connection: {conn_name}") - def _roundtrip(self, test_frame1): - self.drop_table("test_frame_roundtrip", self.conn) - assert self.pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4 - result = self.pandasSQL.read_query("SELECT * FROM test_frame_roundtrip") + return func - result.set_index("level_0", inplace=True) - # result.index.astype(int) - result.index.name = None +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable_iris) +def test_read_sql_iris_parameter(conn, request, sql_strings, flavor): + conn_name = conn + if "engine" in conn: + request.node.add_marker( + pytest.xfail(reason="fails and hangs forever with engine") + ) - tm.assert_frame_equal(result, test_frame1) + conn = request.getfixturevalue(conn) + query = sql_strings["read_parameters"][flavor(conn_name)] + params = ("Iris-setosa", 5.1) + pandasSQL = pandasSQL_builder(conn) + iris_frame = pandasSQL.read_query(query, params=params) + check_iris_frame(iris_frame) - def _execute_sql(self): - # drop_sql = "DROP TABLE IF EXISTS test" # should already be done - iris_results = self.pandasSQL.execute("SELECT * FROM iris") - row = iris_results.fetchone() - tm.equalContents(row, [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]) - def _to_sql_save_index(self): - df = DataFrame.from_records( - [(1, 2.1, "line1"), (2, 1.5, "line2")], columns=["A", "B", "C"], index=["A"] +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable_iris) +def test_read_sql_iris_named_parameter(conn, request, sql_strings, flavor): + conn_name = conn + if "engine" in conn: + request.node.add_marker( + pytest.xfail(reason="fails and hangs forever with engine") ) - assert self.pandasSQL.to_sql(df, "test_to_sql_saves_index") == 2 - ix_cols = self._get_index_columns("test_to_sql_saves_index") - assert ix_cols == [["A"]] - - def _transaction_test(self): - with self.pandasSQL.run_transaction() as trans: - stmt = "CREATE TABLE test_trans (A INT, B TEXT)" - if isinstance(self.pandasSQL, SQLiteDatabase): - trans.execute(stmt) - else: - from sqlalchemy import text - - stmt = text(stmt) - trans.execute(stmt) - - class DummyException(Exception): - pass + conn = request.getfixturevalue(conn) + query = sql_strings["read_named_parameters"][flavor(conn_name)] + params = {"name": "Iris-setosa", "length": 5.1} + pandasSQL = pandasSQL_builder(conn) + iris_frame = pandasSQL.read_query(query, params=params) + check_iris_frame(iris_frame) - # Make sure when transaction is rolled back, no rows get inserted - ins_sql = "INSERT INTO test_trans (A,B) VALUES (1, 'blah')" - if isinstance(self.pandasSQL, SQLDatabase): - from sqlalchemy import text - ins_sql = text(ins_sql) - try: - with self.pandasSQL.run_transaction() as trans: - trans.execute(ins_sql) - raise DummyException("error") - except DummyException: - # ignore raised exception - pass - res = self.pandasSQL.read_query("SELECT * FROM test_trans") - assert len(res) == 0 +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable_iris) +def test_read_sql_iris_no_parameter_with_percent(conn, request, sql_strings, flavor): + conn_name = conn + conn = request.getfixturevalue(conn) - # Make sure when transaction is committed, rows do get inserted - with self.pandasSQL.run_transaction() as trans: - trans.execute(ins_sql) - res2 = self.pandasSQL.read_query("SELECT * FROM test_trans") - assert len(res2) == 1 + query = sql_strings["read_no_parameters_with_percent"][flavor(conn_name)] + pandasSQL = pandasSQL_builder(conn) + iris_frame = pandasSQL.read_query(query, params=None) + check_iris_frame(iris_frame) # ----------------------------------------------------------------------------- @@ -2827,6 +2763,47 @@ def test_transactions(conn, request): trans.execute(stmt) +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +def test_transaction_rollback(conn, request): + conn = request.getfixturevalue(conn) + pandasSQL = pandasSQL_builder(conn) + with pandasSQL.run_transaction() as trans: + stmt = "CREATE TABLE test_trans (A INT, B TEXT)" + if isinstance(pandasSQL, SQLiteDatabase): + trans.execute(stmt) + else: + from sqlalchemy import text + + stmt = text(stmt) + trans.execute(stmt) + + class DummyException(Exception): + pass + + # Make sure when transaction is rolled back, no rows get inserted + ins_sql = "INSERT INTO test_trans (A,B) VALUES (1, 'blah')" + if isinstance(pandasSQL, SQLDatabase): + from sqlalchemy import text + + ins_sql = text(ins_sql) + try: + with pandasSQL.run_transaction() as trans: + trans.execute(ins_sql) + raise DummyException("error") + except DummyException: + # ignore raised exception + pass + res = pandasSQL.read_query("SELECT * FROM test_trans") + assert len(res) == 0 + + # Make sure when transaction is committed, rows do get inserted + with pandasSQL.run_transaction() as trans: + trans.execute(ins_sql) + res2 = pandasSQL.read_query("SELECT * FROM test_trans") + assert len(res2) == 1 + + @pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_get_schema_create_table(conn, request, test_frame3): @@ -3112,6 +3089,22 @@ def test_invalid_engine(conn, request, test_frame1): pandasSQL.to_sql(test_frame1, "test_frame1", engine="bad_engine") +@pytest.mark.skip("Couldn't get this to work?") +@pytest.mark.db +@pytest.mark.parametrize("conn", all_connectable) +def test_to_sql_with_sql_engine(conn, request, test_frame1): + """`to_sql` with the `engine` param""" + # mostly copied from this class's `_to_sql()` method + conn = request.getfixturevalue(conn) + pandasSQL = pandasSQL_builder(conn) + assert pandasSQL.to_sql(test_frame1, "test_frame1", engine="auto") == 4 + assert pandasSQL.has_table("test_frame1") + + num_entries = len(test_frame1) + num_rows = count_rows(conn, "test_frame1") + assert num_rows == num_entries + + @pytest.mark.skip("Couldn't get this to work?") @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) From 97a4eba833275eefd66d8e6ccdd4993567533d73 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Sep 2023 12:13:00 -0400 Subject: [PATCH 10/38] factory func --- pandas/tests/io/test_sql.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 34dc7e98f7d38..0b78b6bf11580 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2256,10 +2256,7 @@ def test_roundtrip(conn, request, test_frame1): conn = request.getfixturevalue(conn) drop_table("test_frame_roundtrip", conn) - if conn_name == "sqlite_buildin": - pandasSQL = sql.SQLiteDatabase(conn) - else: - pandasSQL = sql.SQLDatabase(conn) + pandasSQL = pandasSQL_builder(conn) assert pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4 result = pandasSQL.read_query("SELECT * FROM test_frame_roundtrip") @@ -2276,10 +2273,7 @@ def test_roundtrip(conn, request, test_frame1): def test_execute_sql(conn, request): conn_name = conn conn = request.getfixturevalue(conn) - if conn_name == "sqlite_buildin_iris": - pandasSQL = sql.SQLiteDatabase(conn) - else: - pandasSQL = sql.SQLDatabase(conn) + pandasSQL = pandasSQL_builder(conn) iris_results = pandasSQL.execute("SELECT * FROM iris") row = iris_results.fetchone() tm.equalContents(row, [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]) @@ -2714,14 +2708,8 @@ def test_to_sql_save_index(conn, request): [(1, 2.1, "line1"), (2, 1.5, "line2")], columns=["A", "B", "C"], index=["A"] ) - if conn_name == "sqlite_buildin": - pandasSQL = sql.SQLiteDatabase(conn) - tbl_name = "test_to_sql_saves_index" - assert pandasSQL.to_sql(df, tbl_name) == 2 - else: - pandasSQL = sql.SQLDatabase(conn) - tbl_name = "test_to_sql_saves_index" - assert pandasSQL.to_sql(df, tbl_name) == 2 + pandasSQL = pandasSQL_builder(conn) + assert pandasSQL.to_sql(df, tbl_name) == 2 if conn_name in {"sqlite_buildin", "sqlite_str"}: ixs = sql.read_sql_query( @@ -2751,10 +2739,8 @@ def test_transactions(conn, request): conn = request.getfixturevalue(conn) stmt = "CREATE TABLE test_trans (A INT, B TEXT)" - if conn_name == "sqlite_buildin": - pandasSQL = sql.SQLiteDatabase(conn) - else: - pandasSQL = sql.SQLDatabase(conn) + pandasSQL = pandasSQL_builder(conn) + if conn_name != "sqlite_buildin": from sqlalchemy import text stmt = text(stmt) From 9557f14af9a57a41293e68b8275d3930ba9b1d10 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Sep 2023 12:49:15 -0400 Subject: [PATCH 11/38] most code cleanups --- pandas/tests/io/test_sql.py | 102 +++++++++++++++--------------------- 1 file changed, 42 insertions(+), 60 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 0b78b6bf11580..9feacd6d97d0b 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -608,8 +608,14 @@ def sqlite_sqlalchemy_memory(iris_path, types_data): @pytest.fixture -def sqlite_buildin_iris(sqlite_buildin, iris_path): +def sqlite_buildin_iris(sqlite_buildin, iris_path, types_data): create_and_load_iris_sqlite3(sqlite_buildin, iris_path) + + for entry in types_data: + entry.pop("DateColWithTz") + types_data = [tuple(entry.values()) for entry in types_data] + + create_and_load_types_sqlite3(sqlite_buildin, types_data) return sqlite_buildin @@ -845,22 +851,36 @@ def sample(pd_table, conn, keys, data_iter): @pytest.mark.db -@pytest.mark.parametrize("conn", mysql_connectable) +@pytest.mark.parametrize("conn", all_connectable_iris) def test_default_type_conversion(conn, request): + conn_name = conn + if conn_name == "sqlite_buildin_iris": + request.node.add_marker( + pytest.mark.xfail( + reason="sqlite_buildin connection does not implement read_sql_table" + ) + ) + conn = request.getfixturevalue(conn) df = sql.read_sql_table("types", conn) assert issubclass(df.FloatCol.dtype.type, np.floating) assert issubclass(df.IntCol.dtype.type, np.integer) - # MySQL has no real BOOL type (it's an alias for TINYINT) - assert issubclass(df.BoolCol.dtype.type, np.integer) + # MySQL/sqlite has no real BOOL type + if "postgresql" in conn_name: + assert issubclass(df.BoolCol.dtype.type, np.bool_) + else: + assert issubclass(df.BoolCol.dtype.type, np.integer) # Int column with NA values stays as float assert issubclass(df.IntColWithNull.dtype.type, np.floating) # Bool column with NA = int column with NA values => becomes float - assert issubclass(df.BoolColWithNull.dtype.type, np.floating) + if "postgresql" in conn_name: + assert issubclass(df.BoolColWithNull.dtype.type, object) + else: + assert issubclass(df.BoolColWithNull.dtype.type, np.floating) @pytest.mark.db @@ -1157,7 +1177,7 @@ def test_read_sql_iris_parameter(conn, request, sql_strings, flavor): conn_name = conn if "engine" in conn: request.node.add_marker( - pytest.xfail(reason="fails and hangs forever with engine") + pytest.mark.xfail(reason="fails and hangs forever with engine") ) conn = request.getfixturevalue(conn) @@ -1174,7 +1194,7 @@ def test_read_sql_iris_named_parameter(conn, request, sql_strings, flavor): conn_name = conn if "engine" in conn: request.node.add_marker( - pytest.xfail(reason="fails and hangs forever with engine") + pytest.mark.xfail(reason="fails and hangs forever with engine") ) conn = request.getfixturevalue(conn) query = sql_strings["read_named_parameters"][flavor(conn_name)] @@ -1363,12 +1383,9 @@ def test_api_execute_sql(conn, request): @pytest.mark.db -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("conn", all_connectable_iris) def test_api_date_parsing(conn, request): conn_name = conn - if conn_name in {"sqlite_buildin", "sqlite_str"}: - pytest.skip("types tables not created in sqlite_buildin or sqlite_str fixture") - conn = request.getfixturevalue(conn) # Test date parsing in read_sql # No Parsing @@ -1423,7 +1440,7 @@ def test_api_date_parsing(conn, request): @pytest.mark.db -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("conn", all_connectable_iris) @pytest.mark.parametrize("error", ["ignore", "raise", "coerce"]) @pytest.mark.parametrize( "read_sql, text, mode", @@ -1442,9 +1459,6 @@ def test_api_custom_dateparsing_error( conn, request, read_sql, text, mode, error, types_data_frame ): conn_name = conn - if conn_name in {"sqlite_buildin", "sqlite_str"}: - pytest.skip("types tables not created in sqlite_buildin or sqlite_str fixture") - conn = request.getfixturevalue(conn) expected = types_data_frame.astype({"DateCol": "datetime64[ns]"}) @@ -1466,13 +1480,9 @@ def test_api_custom_dateparsing_error( @pytest.mark.db -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("conn", all_connectable_iris) def test_api_date_and_index(conn, request): # Test case where same column appears in parse_date and index_col - conn_name = conn - if conn_name in {"sqlite_buildin", "sqlite_str"}: - pytest.skip("types tables not created in sqlite_buildin or sqlite_str fixture") - conn = request.getfixturevalue(conn) df = sql.read_sql_query( "SELECT * FROM types", @@ -2154,7 +2164,7 @@ def close(self): @pytest.mark.db -def test_read_sql_delegate(sqlite_buildin_iris): +def test_sqlite_read_sql_delegate(sqlite_buildin_iris): conn = sqlite_buildin_iris iris_frame1 = sql.read_sql_query("SELECT * FROM iris", conn) iris_frame2 = sql.read_sql("SELECT * FROM iris", conn) @@ -2220,7 +2230,7 @@ def test_drop_table(conn, request): pytest.skip("sqlite_str has no inspection system") elif "engine" in conn: request.node.add_marker( - pytest.xfail(reason="fails and hangs forever with engine") + pytest.mark.xfail(reason="fails and hangs forever with engine") ) from sqlalchemy import inspect @@ -2245,12 +2255,11 @@ def test_drop_table(conn, request): @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_roundtrip(conn, request, test_frame1): - conn_name = conn if conn == "sqlite_str": pytest.skip("sqlite_str has no inspection system") elif "engine" in conn: request.node.add_marker( - pytest.xfail(reason="fails and hangs forever with engine") + pytest.mark.xfail(reason="fails and hangs forever with engine") ) conn = request.getfixturevalue(conn) @@ -2271,7 +2280,6 @@ def test_roundtrip(conn, request, test_frame1): @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_execute_sql(conn, request): - conn_name = conn conn = request.getfixturevalue(conn) pandasSQL = pandasSQL_builder(conn) iris_results = pandasSQL.execute("SELECT * FROM iris") @@ -2281,7 +2289,7 @@ def test_execute_sql(conn, request): @pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) -def test_read_table(conn, request): +def test_sqlalchemy_read_table(conn, request): conn = request.getfixturevalue(conn) iris_frame = sql.read_sql_table("iris", con=conn) check_iris_frame(iris_frame) @@ -2289,7 +2297,7 @@ def test_read_table(conn, request): @pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) -def test_read_table_columns(conn, request): +def test_sqlalchemy_read_table_columns(conn, request): conn = request.getfixturevalue(conn) iris_frame = sql.read_sql_table( "iris", con=conn, columns=["SepalLength", "SepalLength"] @@ -2308,13 +2316,13 @@ def test_read_table_absent_raises(conn, request): @pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) -def test_default_type_conversion(conn, request): +def test_sqlalchemy_default_type_conversion(conn, request): conn_name = conn if conn_name == "sqlite_str": pytest.skip("types tables not created in sqlite_str fixture") elif "mysql" in conn_name or "sqlite" in conn_name: request.node.add_marker( - pytest.xfail(reason="boolean dtype not inferred properly") + pytest.mark.xfail(reason="boolean dtype not inferred properly") ) conn = request.getfixturevalue(conn) @@ -2350,7 +2358,7 @@ def test_default_date_load(conn, request): pytest.skip("types tables not created in sqlite_str fixture") elif "sqlite" in conn_name: request.node.add_marker( - pytest.xfail(reason="sqlite does not read date properly") + pytest.mark.xfail(reason="sqlite does not read date properly") ) conn = request.getfixturevalue(conn) @@ -2699,7 +2707,7 @@ def test_nan_string(conn, request): def test_to_sql_save_index(conn, request): if "engine" in conn: request.node.add_marker( - pytest.xfail(reason="fails and hangs forever with engine") + pytest.mark.xfail(reason="fails and hangs forever with engine") ) conn_name = conn @@ -2709,6 +2717,7 @@ def test_to_sql_save_index(conn, request): ) pandasSQL = pandasSQL_builder(conn) + tbl_name = "test_to_sql_saves_index" assert pandasSQL.to_sql(df, tbl_name) == 2 if conn_name in {"sqlite_buildin", "sqlite_str"}: @@ -2739,7 +2748,7 @@ def test_transactions(conn, request): conn = request.getfixturevalue(conn) stmt = "CREATE TABLE test_trans (A INT, B TEXT)" - pandasSQL = pandasSQL_builder(conn) + pandasSQL = pandasSQL_builder(conn) if conn_name != "sqlite_buildin": from sqlalchemy import text @@ -2798,7 +2807,7 @@ def test_get_schema_create_table(conn, request, test_frame3): # mismatch) if conn == "sqlite_str": request.node.add_marker( - pytest.xfail(reason="test does not support sqlite_str fixture") + pytest.mark.xfail(reason="test does not support sqlite_str fixture") ) conn = request.getfixturevalue(conn) @@ -3364,33 +3373,6 @@ def test_keyword_deprecation(sqlite_sqlalchemy_memory): df.to_sql("example", conn, None, if_exists="replace") -@pytest.mark.db -def test_default_type_conversion(sqlite_sqlalchemy_memory): - conn = sqlite_sqlalchemy_memory - df = sql.read_sql_table("types", conn) - - assert issubclass(df.FloatCol.dtype.type, np.floating) - assert issubclass(df.IntCol.dtype.type, np.integer) - - # sqlite has no boolean type, so integer type is returned - assert issubclass(df.BoolCol.dtype.type, np.integer) - - # Int column with NA values stays as float - assert issubclass(df.IntColWithNull.dtype.type, np.floating) - - # Non-native Bool column with NA values stays as float - assert issubclass(df.BoolColWithNull.dtype.type, np.floating) - - -@pytest.mark.db -def test_default_date_load(sqlite_sqlalchemy_memory): - conn = sqlite_sqlalchemy_memory - df = sql.read_sql_table("types", conn) - - # IMPORTANT - sqlite has no native date type, so shouldn't parse, but - assert not issubclass(df.DateCol.dtype.type, np.datetime64) - - @pytest.mark.db def test_bigint_warning(sqlite_sqlalchemy_memory): conn = sqlite_sqlalchemy_memory From e8c9b6d81dd50456105bd273ba47324be59d9d85 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Sep 2023 15:25:26 -0400 Subject: [PATCH 12/38] removed breakpoint; passing tests --- pandas/tests/io/test_sql.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 9feacd6d97d0b..7c0e217d968fa 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -3100,30 +3100,43 @@ def test_to_sql_with_sql_engine(conn, request, test_frame1): assert num_rows == num_entries -@pytest.mark.skip("Couldn't get this to work?") @pytest.mark.db -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_options_sqlalchemy(conn, request, test_frame1): + if "engine" in conn: + pytest.skip("hangs forever with engine arguments") + elif conn == "sqlite_str": + request.node.add_marker(pytest.mark.xfail(reason="broken with sqlite_str")) # use the set option conn = request.getfixturevalue(conn) with pd.option_context("io.sql.engine", "sqlalchemy"): pandasSQL = pandasSQL_builder(conn) - assert pandasSQL.to_sql(test_frame1, "test_frame1", engine="auto") == 4 + assert pandasSQL.to_sql(test_frame1, "test_frame1") == 4 - breakpoint() assert pandasSQL.has_table("test_frame1") num_entries = len(test_frame1) num_rows = count_rows(conn, "test_frame1") assert num_rows == num_entries -@pytest.mark.skip("Couldn't get this to work?") @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_options_auto(conn, request, test_frame1): + if "engine" in conn: + pytest.skip("hangs forever with engine arguments") + elif conn == "sqlite_str": + request.node.add_marker(pytest.mark.xfail(reason="broken with sqlite_str")) + # use the set option + conn = request.getfixturevalue(conn) with pd.option_context("io.sql.engine", "auto"): - self._to_sql_with_sql_engine(test_frame1) + pandasSQL = pandasSQL_builder(conn) + assert pandasSQL.to_sql(test_frame1, "test_frame1") == 4 + + assert pandasSQL.has_table("test_frame1") + num_entries = len(test_frame1) + num_rows = count_rows(conn, "test_frame1") + assert num_rows == num_entries def test_options_get_engine(): From ea1964d9f8a2bfa49eae4e632f4fd3413fd2d417 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Sep 2023 15:34:22 -0400 Subject: [PATCH 13/38] fixes --- pandas/tests/io/test_sql.py | 84 ++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 7c0e217d968fa..88bfc3fcca5ab 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1176,9 +1176,7 @@ def func(conn_name): def test_read_sql_iris_parameter(conn, request, sql_strings, flavor): conn_name = conn if "engine" in conn: - request.node.add_marker( - pytest.mark.xfail(reason="fails and hangs forever with engine") - ) + pytest.skip(reason="can hang forever with engine") conn = request.getfixturevalue(conn) query = sql_strings["read_parameters"][flavor(conn_name)] @@ -1193,9 +1191,8 @@ def test_read_sql_iris_parameter(conn, request, sql_strings, flavor): def test_read_sql_iris_named_parameter(conn, request, sql_strings, flavor): conn_name = conn if "engine" in conn: - request.node.add_marker( - pytest.mark.xfail(reason="fails and hangs forever with engine") - ) + pytest.skip(reason="can hang forever with engine") + conn = request.getfixturevalue(conn) query = sql_strings["read_named_parameters"][flavor(conn_name)] params = {"name": "Iris-setosa", "length": 5.1} @@ -1207,6 +1204,9 @@ def test_read_sql_iris_named_parameter(conn, request, sql_strings, flavor): @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_read_sql_iris_no_parameter_with_percent(conn, request, sql_strings, flavor): + if "mysql" in conn or "postgresql" in conn: + request.node.add_marker(pytest.mark.xfail(reason="broken test")) + conn_name = conn conn = request.getfixturevalue(conn) @@ -1460,6 +1460,10 @@ def test_api_custom_dateparsing_error( ): conn_name = conn conn = request.getfixturevalue(conn) + if text == "types" and conn_name == "sqlite_buildin_iris": + request.node.add_marker( + pytest.mark.xfail(reason="failing combination of arguments") + ) expected = types_data_frame.astype({"DateCol": "datetime64[ns]"}) @@ -1914,8 +1918,15 @@ def test_read_table_index_col(conn, request, test_frame1): @pytest.mark.db -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("conn", all_connectable_iris) def test_read_sql_delegate(conn, request): + if conn == "sqlite_buildin_iris": + request.node.add_marker( + pytest.mark.xfail( + reason="sqlite_buildin connection does not implement read_sql_table" + ) + ) + conn = request.getfixturevalue(conn) iris_frame1 = sql.read_sql_query("SELECT * FROM iris", conn) iris_frame2 = sql.read_sql("SELECT * FROM iris", conn) @@ -2229,9 +2240,7 @@ def test_drop_table(conn, request): if conn == "sqlite_str": pytest.skip("sqlite_str has no inspection system") elif "engine" in conn: - request.node.add_marker( - pytest.mark.xfail(reason="fails and hangs forever with engine") - ) + pytest.skip(reason="fails and hangs forever with engine") from sqlalchemy import inspect @@ -2258,9 +2267,7 @@ def test_roundtrip(conn, request, test_frame1): if conn == "sqlite_str": pytest.skip("sqlite_str has no inspection system") elif "engine" in conn: - request.node.add_marker( - pytest.mark.xfail(reason="fails and hangs forever with engine") - ) + pytest.skip(reason="fails and hangs forever with engine") conn = request.getfixturevalue(conn) drop_table("test_frame_roundtrip", conn) @@ -2706,9 +2713,7 @@ def test_nan_string(conn, request): @pytest.mark.parametrize("conn", all_connectable) def test_to_sql_save_index(conn, request): if "engine" in conn: - request.node.add_marker( - pytest.mark.xfail(reason="fails and hangs forever with engine") - ) + pytest.skip(reason="fails and hangs forever with engine") conn_name = conn conn = request.getfixturevalue(conn) @@ -2761,6 +2766,9 @@ def test_transactions(conn, request): @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_transaction_rollback(conn, request): + if "engine" in conn: + pytest.skip(reason="fails and hangs forever with engine") + conn = request.getfixturevalue(conn) pandasSQL = pandasSQL_builder(conn) with pandasSQL.run_transaction() as trans: @@ -2773,30 +2781,30 @@ def test_transaction_rollback(conn, request): stmt = text(stmt) trans.execute(stmt) - class DummyException(Exception): - pass + class DummyException(Exception): + pass - # Make sure when transaction is rolled back, no rows get inserted - ins_sql = "INSERT INTO test_trans (A,B) VALUES (1, 'blah')" - if isinstance(pandasSQL, SQLDatabase): - from sqlalchemy import text + # Make sure when transaction is rolled back, no rows get inserted + ins_sql = "INSERT INTO test_trans (A,B) VALUES (1, 'blah')" + if isinstance(pandasSQL, SQLDatabase): + from sqlalchemy import text - ins_sql = text(ins_sql) - try: - with pandasSQL.run_transaction() as trans: - trans.execute(ins_sql) - raise DummyException("error") - except DummyException: - # ignore raised exception - pass - res = pandasSQL.read_query("SELECT * FROM test_trans") - assert len(res) == 0 - - # Make sure when transaction is committed, rows do get inserted + ins_sql = text(ins_sql) + try: with pandasSQL.run_transaction() as trans: trans.execute(ins_sql) - res2 = pandasSQL.read_query("SELECT * FROM test_trans") - assert len(res2) == 1 + raise DummyException("error") + except DummyException: + # ignore raised exception + pass + res = pandasSQL.read_query("SELECT * FROM test_trans") + assert len(res) == 0 + + # Make sure when transaction is committed, rows do get inserted + with pandasSQL.run_transaction() as trans: + trans.execute(ins_sql) + res2 = pandasSQL.read_query("SELECT * FROM test_trans") + assert len(res2) == 1 @pytest.mark.db @@ -3104,7 +3112,7 @@ def test_to_sql_with_sql_engine(conn, request, test_frame1): @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_options_sqlalchemy(conn, request, test_frame1): if "engine" in conn: - pytest.skip("hangs forever with engine arguments") + pytest.skip("fails and hangs forever with engine") elif conn == "sqlite_str": request.node.add_marker(pytest.mark.xfail(reason="broken with sqlite_str")) # use the set option @@ -3123,7 +3131,7 @@ def test_options_sqlalchemy(conn, request, test_frame1): @pytest.mark.parametrize("conn", all_connectable) def test_options_auto(conn, request, test_frame1): if "engine" in conn: - pytest.skip("hangs forever with engine arguments") + pytest.skip("fails and hangs forever with engine") elif conn == "sqlite_str": request.node.add_marker(pytest.mark.xfail(reason="broken with sqlite_str")) From 60b77872795b1e426e6a6c76f930aa05e476adc2 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Sep 2023 17:40:19 -0400 Subject: [PATCH 14/38] fix when missing SQLAlchemy --- pandas/tests/io/test_sql.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 88bfc3fcca5ab..3a781d1563853 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1986,7 +1986,7 @@ def test_warning_case_insensitive_table_name(conn, request, test_frame1): @pytest.mark.db -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_sqlalchemy_type_mapping(conn, request): conn = request.getfixturevalue(conn) from sqlalchemy import TIMESTAMP @@ -2002,7 +2002,7 @@ def test_sqlalchemy_type_mapping(conn, request): @pytest.mark.db -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("conn", sqlalchemy_connectable) @pytest.mark.parametrize( "integer, expected", [ @@ -2049,6 +2049,7 @@ def test_sqlalchemy_integer_overload_mapping(conn, request, integer): @pytest.mark.db +@pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="fails without SQLAlchemy") @pytest.mark.parametrize("conn", all_connectable) def test_database_uri_string(conn, request, test_frame1): conn = request.getfixturevalue(conn) @@ -2070,6 +2071,7 @@ def test_database_uri_string(conn, request, test_frame1): @td.skip_if_installed("pg8000") +@pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="fails without SQLAlchemy") @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_pg8000_sqlalchemy_passthrough_error(conn, request): @@ -2218,10 +2220,10 @@ def test_create_table(conn, request): if conn == "sqlite_str": pytest.skip("sqlite_str has no inspection system") - from sqlalchemy import inspect - conn = request.getfixturevalue(conn) + from sqlalchemy import inspect + temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL: assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 @@ -2887,6 +2889,9 @@ def test_notna_dtype(conn, request): if conn == "sqlite_str": pytest.skip("sqlite_str has no inspection system") + conn_name = conn + conn = request.getfixturevalue(conn) + from sqlalchemy import ( Boolean, DateTime, @@ -2895,9 +2900,6 @@ def test_notna_dtype(conn, request): ) from sqlalchemy.schema import MetaData - conn_name = conn - conn = request.getfixturevalue(conn) - cols = { "Bool": Series([True, None]), "Date": Series([datetime(2012, 5, 1), None]), @@ -2925,6 +2927,8 @@ def test_double_precision(conn, request): if conn == "sqlite_str": pytest.skip("sqlite_str has no inspection system") + conn = request.getfixturevalue(conn) + from sqlalchemy import ( BigInteger, Float, @@ -2932,7 +2936,6 @@ def test_double_precision(conn, request): ) from sqlalchemy.schema import MetaData - conn = request.getfixturevalue(conn) V = 1.23456789101112131415 df = DataFrame( @@ -2974,6 +2977,8 @@ def test_double_precision(conn, request): @pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_connectable_issue_example(conn, request): + conn = request.getfixturevalue(conn) + # This tests the example raised in issue # https://github.com/pandas-dev/pandas/issues/10104 from sqlalchemy.engine import Engine @@ -2999,7 +3004,6 @@ def main(connectable): else: test_connectable(connectable) - conn = request.getfixturevalue(conn) assert ( DataFrame({"test_foo_data": [0, 1, 2]}).to_sql(name="test_foo_data", con=conn) == 3 @@ -3043,6 +3047,8 @@ def test_to_sql_with_negative_npinf(conn, request, input): @pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_temporary_table(conn, request): + conn = request.getfixturevalue(conn) + if conn == "sqlite_str": pytest.skip("test does not work with str connection") @@ -3057,7 +3063,6 @@ def test_temporary_table(conn, request): declarative_base, ) - conn = request.getfixturevalue(conn) test_data = "Hello, World!" expected = DataFrame({"spam": [test_data]}) Base = declarative_base() @@ -3147,6 +3152,8 @@ def test_options_auto(conn, request, test_frame1): assert num_rows == num_entries +@pytest.mark.db +@pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="fails without SQLAlchemy") def test_options_get_engine(): assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine) From c1c13b47b743151ea6a6edfc7290cbb3febed7a2 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Sep 2023 18:33:08 -0400 Subject: [PATCH 15/38] more fixups when no SQLAlchemy --- pandas/tests/io/test_sql.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 3a781d1563853..9e9d485cdfb9f 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2035,7 +2035,7 @@ def test_sqlalchemy_integer_mapping(conn, request, integer, expected): @pytest.mark.db -@pytest.mark.parametrize("conn", all_connectable) +@pytest.mark.parametrize("conn", sqlalchemy_connectable) @pytest.mark.parametrize("integer", ["uint64", "UInt64"]) def test_sqlalchemy_integer_overload_mapping(conn, request, integer): conn = request.getfixturevalue(conn) @@ -2244,10 +2244,10 @@ def test_drop_table(conn, request): elif "engine" in conn: pytest.skip(reason="fails and hangs forever with engine") - from sqlalchemy import inspect - conn = request.getfixturevalue(conn) + from sqlalchemy import inspect + temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) pandasSQL = sql.SQLDatabase(conn) assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 @@ -2847,13 +2847,14 @@ def test_dtype(conn, request): if conn == "sqlite_str": pytest.skip("sqlite_str has no inspection system") + conn = request.getfixturevalue(conn) + from sqlalchemy import ( TEXT, String, ) from sqlalchemy.schema import MetaData - conn = request.getfixturevalue(conn) cols = ["A", "B"] data = [(0.8, True), (0.9, None)] df = DataFrame(data, columns=cols) From 7223810c8c8223b3627ce47713f29e3d11f9bb1b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 9 Sep 2023 23:48:14 -0400 Subject: [PATCH 16/38] fixups --- pandas/tests/io/test_sql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 9e9d485cdfb9f..9152e2faf3cc5 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -3048,11 +3048,11 @@ def test_to_sql_with_negative_npinf(conn, request, input): @pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_temporary_table(conn, request): - conn = request.getfixturevalue(conn) - if conn == "sqlite_str": pytest.skip("test does not work with str connection") + conn = request.getfixturevalue(conn) + from sqlalchemy import ( Column, Integer, @@ -3120,7 +3120,7 @@ def test_options_sqlalchemy(conn, request, test_frame1): if "engine" in conn: pytest.skip("fails and hangs forever with engine") elif conn == "sqlite_str": - request.node.add_marker(pytest.mark.xfail(reason="broken with sqlite_str")) + pytest.skip("may hang on old versions of sqlite") # use the set option conn = request.getfixturevalue(conn) with pd.option_context("io.sql.engine", "sqlalchemy"): From 21a93214118f5b0c4e362b5dd610767947c27a8d Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 10 Sep 2023 08:03:53 -0400 Subject: [PATCH 17/38] xfail -> skip --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 9152e2faf3cc5..3b25ba5b8d99d 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -3139,7 +3139,7 @@ def test_options_auto(conn, request, test_frame1): if "engine" in conn: pytest.skip("fails and hangs forever with engine") elif conn == "sqlite_str": - request.node.add_marker(pytest.mark.xfail(reason="broken with sqlite_str")) + pytest.skip("may hang on old versions of sqlite") # use the set option conn = request.getfixturevalue(conn) From 63a997335d4b3655ae0c8d34b814511fd8b00662 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 10 Sep 2023 10:10:55 -0400 Subject: [PATCH 18/38] sqlite fixture use transaction for cleanup --- pandas/tests/io/test_sql.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 3b25ba5b8d99d..2b2c43b58f61a 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -535,10 +535,13 @@ def sqlite_engine(sqlite_str, iris_path, types_data): create_and_load_types(engine, types_data, "sqlite") yield engine - for view in get_all_views(engine): - drop_view(view, engine) - for tbl in get_all_tables(engine): - drop_table(tbl, engine) + with engine.connect() as conn: + with conn.begin(): + for view in get_all_views(engine): + drop_view(view, conn) + for tbl in get_all_tables(engine): + drop_table(tbl, conn) + engine.dispose() @@ -1947,6 +1950,7 @@ def test_not_reflect_all_tables(sqlite_conn): text("CREATE TABLE invalid (x INTEGER, y UNKNOWN);"), text("CREATE TABLE other_table (x INTEGER, y INTEGER);"), ] + for query in query_list: if isinstance(conn, Engine): with conn.connect() as conn: From 6d3ab3747ac7d2870ddb4e53e709f6d180956d7b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 10 Sep 2023 20:18:40 -0400 Subject: [PATCH 19/38] verbose test for hangs --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 48ef21686a26f..a2f25776129ef 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml" -PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -v -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 60b1c5463cb2420da6b24788e5815a75793fef0e Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sun, 10 Sep 2023 22:10:05 -0400 Subject: [PATCH 20/38] try skipping sqlite-sqlalchemy-memory on rollback test --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 2b2c43b58f61a..32c34c0713dc7 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2772,7 +2772,7 @@ def test_transactions(conn, request): @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_transaction_rollback(conn, request): - if "engine" in conn: + if "engine" in conn or conn == "sqlite_sqlalchemy_memory": pytest.skip(reason="fails and hangs forever with engine") conn = request.getfixturevalue(conn) From 204bc47b1d1d134faa7e8a5d5ac51418a4e6fa19 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 11 Sep 2023 07:59:49 -0400 Subject: [PATCH 21/38] sqlite sqlaclchemy memory cleanup --- pandas/tests/io/test_sql.py | 46 +++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 32c34c0713dc7..91a675f0d2d22 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -589,7 +589,7 @@ def sqlite_buildin(): @pytest.fixture -def sqlite_sqlalchemy_memory(iris_path, types_data): +def sqlite_sqlalchemy_memory_engine(iris_path, types_data): sqlalchemy = pytest.importorskip("sqlalchemy") engine = sqlalchemy.create_engine("sqlite:///:memory:") @@ -604,10 +604,12 @@ def sqlite_sqlalchemy_memory(iris_path, types_data): create_and_load_types(engine, types_data, "sqlite") yield engine - for view in get_all_views(engine): - drop_view(view, engine) - for tbl in get_all_tables(engine): - drop_table(tbl, engine) + with engine.connect() as conn: + with conn.begin(): + for view in get_all_views(engine): + drop_view(view, engine) + for tbl in get_all_tables(engine): + drop_table(tbl, engine) @pytest.fixture @@ -653,12 +655,12 @@ def sqlite_buildin_iris(sqlite_buildin, iris_path, types_data): all_connectable = sqlalchemy_connectable + [ "sqlite_buildin", - "sqlite_sqlalchemy_memory", + "sqlite_sqlalchemy_memory_engine", ] all_connectable_iris = sqlalchemy_connectable_iris + [ "sqlite_buildin_iris", - "sqlite_sqlalchemy_memory", + "sqlite_sqlalchemy_memory_engine", ] @@ -2772,7 +2774,7 @@ def test_transactions(conn, request): @pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_transaction_rollback(conn, request): - if "engine" in conn or conn == "sqlite_sqlalchemy_memory": + if "engine" in conn: pytest.skip(reason="fails and hangs forever with engine") conn = request.getfixturevalue(conn) @@ -3392,8 +3394,8 @@ def test_read_sql_dtype(conn, request, func, dtype_backend): @pytest.mark.db -def test_keyword_deprecation(sqlite_sqlalchemy_memory): - conn = sqlite_sqlalchemy_memory +def test_keyword_deprecation(sqlite_sqlalchemy_memory_engine): + conn = sqlite_sqlalchemy_memory_engine # GH 54397 msg = ( "Starting with pandas version 3.0 all arguments of to_sql except for the " @@ -3407,8 +3409,8 @@ def test_keyword_deprecation(sqlite_sqlalchemy_memory): @pytest.mark.db -def test_bigint_warning(sqlite_sqlalchemy_memory): - conn = sqlite_sqlalchemy_memory +def test_bigint_warning(sqlite_sqlalchemy_memory_engine): + conn = sqlite_sqlalchemy_memory_engine # test no warning for BIGINT (to support int64) is raised (GH7433) df = DataFrame({"a": [1, 2]}, dtype="int64") assert df.to_sql(name="test_bigintwarning", con=conn, index=False) == 2 @@ -3418,16 +3420,16 @@ def test_bigint_warning(sqlite_sqlalchemy_memory): @pytest.mark.db -def test_valueerror_exception(sqlite_sqlalchemy_memory): - conn = sqlite_sqlalchemy_memory +def test_valueerror_exception(sqlite_sqlalchemy_memory_engine): + conn = sqlite_sqlalchemy_memory_engine df = DataFrame({"col1": [1, 2], "col2": [3, 4]}) with pytest.raises(ValueError, match="Empty table name specified"): df.to_sql(name="", con=conn, if_exists="replace", index=False) @pytest.mark.db -def test_row_object_is_named_tuple(sqlite_sqlalchemy_memory): - conn = sqlite_sqlalchemy_memory +def test_row_object_is_named_tuple(sqlite_sqlalchemy_memory_engine): + conn = sqlite_sqlalchemy_memory_engine # GH 40682 # Test for the is_named_tuple() function # Placed here due to its usage of sqlalchemy @@ -3466,8 +3468,8 @@ class Test(BaseModel): @pytest.mark.db -def test_read_sql_string_inference(sqlite_sqlalchemy_memory): - conn = sqlite_sqlalchemy_memory +def test_read_sql_string_inference(sqlite_sqlalchemy_memory_engine): + conn = sqlite_sqlalchemy_memory_engine # GH#54430 pytest.importorskip("pyarrow") table = "test" @@ -3486,8 +3488,8 @@ def test_read_sql_string_inference(sqlite_sqlalchemy_memory): @pytest.mark.db -def test_roundtripping_datetimes(sqlite_sqlalchemy_memory): - conn = sqlite_sqlalchemy_memory +def test_roundtripping_datetimes(sqlite_sqlalchemy_memory_engine): + conn = sqlite_sqlalchemy_memory_engine # GH#54877 df = DataFrame({"t": [datetime(2020, 12, 31, 12)]}, dtype="datetime64[ns]") df.to_sql("test", conn, if_exists="replace", index=False) @@ -3607,8 +3609,8 @@ def test_self_join_date_columns(postgresql_psycopg2_engine): @pytest.mark.db -def test_create_and_drop_table(sqlite_sqlalchemy_memory): - conn = sqlite_sqlalchemy_memory +def test_create_and_drop_table(sqlite_sqlalchemy_memory_engine): + conn = sqlite_sqlalchemy_memory_engine temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) pandasSQL = sql.SQLDatabase(conn) From eac723be404634fbdecabdb50c59ffd540fb8683 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 11 Sep 2023 09:38:24 -0400 Subject: [PATCH 22/38] revert verbose logging in tests --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index a2f25776129ef..48ef21686a26f 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml" -PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -v -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 3accbe4935cb8b1d624f7a212c3463994d949e3e Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 11 Sep 2023 11:29:39 -0400 Subject: [PATCH 23/38] mark all db tests --- pandas/tests/io/test_sql.py | 140 +----------------------------------- 1 file changed, 3 insertions(+), 137 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 91a675f0d2d22..c84bf8b41255a 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -77,6 +77,9 @@ SQLALCHEMY_INSTALLED = False +pytestmark = [pytest.mark.db] + + @pytest.fixture def sql_strings(): return { @@ -664,7 +667,6 @@ def sqlite_buildin_iris(sqlite_buildin, iris_path, types_data): ] -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_dataframe_to_sql(conn, test_frame1, request): # GH 51086 if conn is sqlite_engine @@ -672,7 +674,6 @@ def test_dataframe_to_sql(conn, test_frame1, request): test_frame1.to_sql(name="test", con=conn, if_exists="append", index=False) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_dataframe_to_sql_empty(conn, test_frame1, request): # GH 51086 if conn is sqlite_engine @@ -681,7 +682,6 @@ def test_dataframe_to_sql_empty(conn, test_frame1, request): empty_df.to_sql(name="test", con=conn, if_exists="append", index=False) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_dataframe_to_sql_arrow_dtypes(conn, request): # GH 52046 @@ -702,7 +702,6 @@ def test_dataframe_to_sql_arrow_dtypes(conn, request): df.to_sql(name="test_arrow", con=conn, if_exists="replace", index=False) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_dataframe_to_sql_arrow_dtypes_missing(conn, request, nulls_fixture): # GH 52046 @@ -718,7 +717,6 @@ def test_dataframe_to_sql_arrow_dtypes_missing(conn, request, nulls_fixture): df.to_sql(name="test_arrow", con=conn, if_exists="replace", index=False) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) @pytest.mark.parametrize("method", [None, "multi"]) def test_to_sql(conn, method, test_frame1, request): @@ -729,7 +727,6 @@ def test_to_sql(conn, method, test_frame1, request): assert count_rows(conn, "test_frame") == len(test_frame1) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) @pytest.mark.parametrize("mode, num_row_coef", [("replace", 1), ("append", 2)]) def test_to_sql_exist(conn, mode, num_row_coef, test_frame1, request): @@ -741,7 +738,6 @@ def test_to_sql_exist(conn, mode, num_row_coef, test_frame1, request): assert count_rows(conn, "test_frame") == num_row_coef * len(test_frame1) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_to_sql_exist_fail(conn, test_frame1, request): conn = request.getfixturevalue(conn) @@ -754,7 +750,6 @@ def test_to_sql_exist_fail(conn, test_frame1, request): pandasSQL.to_sql(test_frame1, "test_frame", if_exists="fail") -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_read_iris_query(conn, request): conn = request.getfixturevalue(conn) @@ -767,7 +762,6 @@ def test_read_iris_query(conn, request): assert "SepalWidth" in iris_frame.columns -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_read_iris_query_chunksize(conn, request): conn = request.getfixturevalue(conn) @@ -780,7 +774,6 @@ def test_read_iris_query_chunksize(conn, request): assert "SepalWidth" in iris_frame.columns -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_read_iris_query_expression_with_parameter(conn, request): conn = request.getfixturevalue(conn) @@ -802,7 +795,6 @@ def test_read_iris_query_expression_with_parameter(conn, request): autoload_con.dispose() -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_read_iris_query_string_with_parameter(conn, request, sql_strings): for db, query in sql_strings["read_parameters"].items(): @@ -815,7 +807,6 @@ def test_read_iris_query_string_with_parameter(conn, request, sql_strings): check_iris_frame(iris_frame) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_read_iris_table(conn, request): # GH 51015 if conn = sqlite_iris_str @@ -826,7 +817,6 @@ def test_read_iris_table(conn, request): check_iris_frame(iris_frame) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_read_iris_table_chunksize(conn, request): conn = request.getfixturevalue(conn) @@ -836,7 +826,6 @@ def test_read_iris_table_chunksize(conn, request): check_iris_frame(iris_frame) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_to_sql_callable(conn, test_frame1, request): conn = request.getfixturevalue(conn) @@ -855,7 +844,6 @@ def sample(pd_table, conn, keys, data_iter): assert count_rows(conn, "test_frame") == len(test_frame1) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_default_type_conversion(conn, request): conn_name = conn @@ -888,7 +876,6 @@ def test_default_type_conversion(conn, request): assert issubclass(df.BoolColWithNull.dtype.type, np.floating) -@pytest.mark.db @pytest.mark.parametrize("conn", mysql_connectable) def test_read_procedure(conn, request): conn = request.getfixturevalue(conn) @@ -926,7 +913,6 @@ def test_read_procedure(conn, request): tm.assert_frame_equal(df, res2) -@pytest.mark.db @pytest.mark.parametrize("conn", postgresql_connectable) @pytest.mark.parametrize("expected_count", [2, "Success!"]) def test_copy_from_callable_insertion_method(conn, expected_count, request): @@ -966,7 +952,6 @@ def psql_insert_copy(table, conn, keys, data_iter): tm.assert_frame_equal(result, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", postgresql_connectable) def test_insertion_method_on_conflict_do_nothing(conn, request): # GH 15988: Example in to_sql docstring @@ -1025,7 +1010,6 @@ def insert_on_conflict(table, conn, keys, data_iter): pandasSQL.drop_table("test_insert_conflict") -@pytest.mark.db @pytest.mark.parametrize("conn", mysql_connectable) def test_insertion_method_on_conflict_update(conn, request): # GH 14553: Example in to_sql docstring @@ -1079,7 +1063,6 @@ def insert_on_conflict(table, conn, keys, data_iter): pandasSQL.drop_table("test_insert_conflict") -@pytest.mark.db @pytest.mark.parametrize("conn", postgresql_connectable) def test_read_view_postgres(conn, request): # GH 52969 @@ -1176,7 +1159,6 @@ def func(conn_name): return func -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_read_sql_iris_parameter(conn, request, sql_strings, flavor): conn_name = conn @@ -1191,7 +1173,6 @@ def test_read_sql_iris_parameter(conn, request, sql_strings, flavor): check_iris_frame(iris_frame) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_read_sql_iris_named_parameter(conn, request, sql_strings, flavor): conn_name = conn @@ -1206,7 +1187,6 @@ def test_read_sql_iris_named_parameter(conn, request, sql_strings, flavor): check_iris_frame(iris_frame) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_read_sql_iris_no_parameter_with_percent(conn, request, sql_strings, flavor): if "mysql" in conn or "postgresql" in conn: @@ -1225,7 +1205,6 @@ def test_read_sql_iris_no_parameter_with_percent(conn, request, sql_strings, fla # -- Testing the public API -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_api_read_sql_view(conn, request): conn = request.getfixturevalue(conn) @@ -1233,7 +1212,6 @@ def test_api_read_sql_view(conn, request): check_iris_frame(iris_frame) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_api_read_sql_with_chunksize_no_result(conn, request): conn = request.getfixturevalue(conn) @@ -1243,7 +1221,6 @@ def test_api_read_sql_with_chunksize_no_result(conn, request): tm.assert_frame_equal(concat(with_batch), without_batch) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_to_sql(conn, request, test_frame1): conn = request.getfixturevalue(conn) @@ -1255,7 +1232,6 @@ def test_api_to_sql(conn, request, test_frame1): assert sql.has_table("test_frame1", conn) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_to_sql_fail(conn, request, test_frame1): conn = request.getfixturevalue(conn) @@ -1271,7 +1247,6 @@ def test_api_to_sql_fail(conn, request, test_frame1): sql.to_sql(test_frame1, "test_frame2", conn, if_exists="fail") -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_to_sql_replace(conn, request, test_frame1): conn = request.getfixturevalue(conn) @@ -1290,7 +1265,6 @@ def test_api_to_sql_replace(conn, request, test_frame1): assert num_rows == num_entries -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_to_sql_append(conn, request, test_frame1): conn = request.getfixturevalue(conn) @@ -1310,7 +1284,6 @@ def test_api_to_sql_append(conn, request, test_frame1): assert num_rows == num_entries -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_to_sql_type_mapping(conn, request, test_frame3): conn = request.getfixturevalue(conn) @@ -1324,7 +1297,6 @@ def test_api_to_sql_type_mapping(conn, request, test_frame3): tm.assert_frame_equal(test_frame3, result) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_to_sql_series(conn, request): conn = request.getfixturevalue(conn) @@ -1338,7 +1310,6 @@ def test_api_to_sql_series(conn, request): tm.assert_frame_equal(s.to_frame(), s2) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_roundtrip(conn, request, test_frame1): conn = request.getfixturevalue(conn) @@ -1357,7 +1328,6 @@ def test_api_roundtrip(conn, request, test_frame1): tm.assert_frame_equal(result, test_frame1) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_roundtrip_chunksize(conn, request, test_frame1): conn = request.getfixturevalue(conn) @@ -1376,7 +1346,6 @@ def test_api_roundtrip_chunksize(conn, request, test_frame1): tm.assert_frame_equal(result, test_frame1) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_api_execute_sql(conn, request): # drop_sql = "DROP TABLE IF EXISTS test" # should already be done @@ -1387,7 +1356,6 @@ def test_api_execute_sql(conn, request): tm.equalContents(row, [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_api_date_parsing(conn, request): conn_name = conn @@ -1444,7 +1412,6 @@ def test_api_date_parsing(conn, request): ] -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) @pytest.mark.parametrize("error", ["ignore", "raise", "coerce"]) @pytest.mark.parametrize( @@ -1488,7 +1455,6 @@ def test_api_custom_dateparsing_error( tm.assert_frame_equal(result, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_api_date_and_index(conn, request): # Test case where same column appears in parse_date and index_col @@ -1504,7 +1470,6 @@ def test_api_date_and_index(conn, request): assert issubclass(df.IntDateCol.dtype.type, np.datetime64) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_timedelta(conn, request): # see #6921 @@ -1521,7 +1486,6 @@ def test_api_timedelta(conn, request): tm.assert_series_equal(result["foo"], df["foo"].view("int64")) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_complex_raises(conn, request): conn = request.getfixturevalue(conn) @@ -1531,7 +1495,6 @@ def test_api_complex_raises(conn, request): assert df.to_sql("test_complex", con=conn) is None -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) @pytest.mark.parametrize( "index_name,index_label,expected", @@ -1564,7 +1527,6 @@ def test_api_to_sql_index_label(conn, request, index_name, index_label, expected assert frame.columns[0] == expected -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_to_sql_index_label_multiindex(conn, request): conn_name = conn @@ -1635,7 +1597,6 @@ def test_api_to_sql_index_label_multiindex(conn, request): ) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_multiindex_roundtrip(conn, request): conn = request.getfixturevalue(conn) @@ -1656,7 +1617,6 @@ def test_api_multiindex_roundtrip(conn, request): tm.assert_frame_equal(df, result, check_index_type=True) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) @pytest.mark.parametrize( "dtype", @@ -1689,7 +1649,6 @@ def test_api_dtype_argument(conn, request, dtype): tm.assert_frame_equal(result, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_integer_col_names(conn, request): conn = request.getfixturevalue(conn) @@ -1697,7 +1656,6 @@ def test_api_integer_col_names(conn, request): sql.to_sql(df, "test_frame_integer_col_names", conn, if_exists="replace") -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_get_schema(conn, request, test_frame1): conn = request.getfixturevalue(conn) @@ -1705,7 +1663,6 @@ def test_api_get_schema(conn, request, test_frame1): assert "CREATE" in create_sql -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_get_schema_with_schema(conn, request, test_frame1): # GH28486 @@ -1714,7 +1671,6 @@ def test_api_get_schema_with_schema(conn, request, test_frame1): assert "CREATE TABLE pypi." in create_sql -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_get_schema_dtypes(conn, request): conn_name = conn @@ -1732,7 +1688,6 @@ def test_api_get_schema_dtypes(conn, request): assert "INTEGER" in create_sql -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_get_schema_keys(conn, request, test_frame1): conn_name = conn @@ -1755,7 +1710,6 @@ def test_api_get_schema_keys(conn, request, test_frame1): assert constraint_sentence in create_sql -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_chunksize_read(conn, request): conn_name = conn @@ -1801,7 +1755,6 @@ def test_api_chunksize_read(conn, request): tm.assert_frame_equal(res1, res3) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_categorical(conn, request): # GH8624 @@ -1826,7 +1779,6 @@ def test_api_categorical(conn, request): tm.assert_frame_equal(res, df) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_unicode_column_name(conn, request): # GH 11431 @@ -1839,7 +1791,6 @@ def test_api_unicode_column_name(conn, request): df.to_sql(name="test_unicode", con=conn, index=False) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_escaped_table_name(conn, request): # GH 13206 @@ -1861,7 +1812,6 @@ def test_api_escaped_table_name(conn, request): tm.assert_frame_equal(res, df) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_api_read_sql_duplicate_columns(conn, request): # GH#53117 @@ -1881,7 +1831,6 @@ def test_api_read_sql_duplicate_columns(conn, request): tm.assert_frame_equal(result, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_read_table_columns(conn, request, test_frame1): # test columns argument in read_table @@ -1898,7 +1847,6 @@ def test_read_table_columns(conn, request, test_frame1): assert result.columns.tolist() == cols -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_read_table_index_col(conn, request, test_frame1): # test columns argument in read_table @@ -1922,7 +1870,6 @@ def test_read_table_index_col(conn, request, test_frame1): assert result.columns.tolist() == ["C", "D"] -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_read_sql_delegate(conn, request): if conn == "sqlite_buildin_iris": @@ -1967,7 +1914,6 @@ def test_not_reflect_all_tables(sqlite_conn): sql.read_sql_query("SELECT * FROM other_table", conn) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_warning_case_insensitive_table_name(conn, request, test_frame1): conn_name = conn @@ -1991,7 +1937,6 @@ def test_warning_case_insensitive_table_name(conn, request, test_frame1): test_frame1.to_sql(name="CaseSensitive", con=conn) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_sqlalchemy_type_mapping(conn, request): conn = request.getfixturevalue(conn) @@ -2007,7 +1952,6 @@ def test_sqlalchemy_type_mapping(conn, request): assert isinstance(table.table.c["time"].type, TIMESTAMP) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) @pytest.mark.parametrize( "integer, expected", @@ -2040,7 +1984,6 @@ def test_sqlalchemy_integer_mapping(conn, request, integer, expected): assert result == expected -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) @pytest.mark.parametrize("integer", ["uint64", "UInt64"]) def test_sqlalchemy_integer_overload_mapping(conn, request, integer): @@ -2054,7 +1997,6 @@ def test_sqlalchemy_integer_overload_mapping(conn, request, integer): sql.SQLTable("test_type", db, frame=df) -@pytest.mark.db @pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="fails without SQLAlchemy") @pytest.mark.parametrize("conn", all_connectable) def test_database_uri_string(conn, request, test_frame1): @@ -2078,7 +2020,6 @@ def test_database_uri_string(conn, request, test_frame1): @td.skip_if_installed("pg8000") @pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="fails without SQLAlchemy") -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_pg8000_sqlalchemy_passthrough_error(conn, request): conn = request.getfixturevalue(conn) @@ -2089,7 +2030,6 @@ def test_pg8000_sqlalchemy_passthrough_error(conn, request): sql.read_sql("select * from table", db_uri) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_query_by_text_obj(conn, request): # WIP : GH10846 @@ -2106,7 +2046,6 @@ def test_query_by_text_obj(conn, request): assert all_names == {"Iris-versicolor"} -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_query_by_select_obj(conn, request): conn = request.getfixturevalue(conn) @@ -2123,7 +2062,6 @@ def test_query_by_select_obj(conn, request): assert all_names == {"Iris-setosa"} -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_column_with_percentage(conn, request): # GH 37157 @@ -2140,7 +2078,6 @@ def test_column_with_percentage(conn, request): tm.assert_frame_equal(res, df) -@pytest.mark.db def test_sql_open_close(test_frame3): # Test if the IO in the database still work if the connection closed # between the writing and reading (as in many real situations). @@ -2155,7 +2092,6 @@ def test_sql_open_close(test_frame3): tm.assert_frame_equal(test_frame3, result) -@pytest.mark.db @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") def test_con_string_import_error(): conn = "mysql://root@localhost/pandas" @@ -2164,7 +2100,6 @@ def test_con_string_import_error(): sql.read_sql("SELECT * FROM iris", conn) -@pytest.mark.db @pytest.mark.skipif(SQLALCHEMY_INSTALLED, reason="SQLAlchemy is installed") def test_con_unknown_dbapi2_class_does_not_error_without_sql_alchemy_installed(): class MockSqliteConnection: @@ -2182,7 +2117,6 @@ def close(self): sql.read_sql("SELECT 1", conn) -@pytest.mark.db def test_sqlite_read_sql_delegate(sqlite_buildin_iris): conn = sqlite_buildin_iris iris_frame1 = sql.read_sql_query("SELECT * FROM iris", conn) @@ -2194,14 +2128,12 @@ def test_sqlite_read_sql_delegate(sqlite_buildin_iris): sql.read_sql("iris", conn) -@pytest.mark.db def test_get_schema2(test_frame1): # without providing a connection object (available for backwards comp) create_sql = sql.get_schema(test_frame1, "test") assert "CREATE" in create_sql -@pytest.mark.db def test_sqlite_type_mapping(sqlite_buildin): # Test Timestamp objects (no datetime64 because of timezone) (GH9085) conn = sqlite_buildin @@ -2220,7 +2152,6 @@ def test_sqlite_type_mapping(sqlite_buildin): # -- Database flavor specific tests -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_create_table(conn, request): if conn == "sqlite_str": @@ -2242,7 +2173,6 @@ def test_create_table(conn, request): pandasSQL.drop_table("temp_frame") -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_drop_table(conn, request): if conn == "sqlite_str": @@ -2269,7 +2199,6 @@ def test_drop_table(conn, request): assert not insp.has_table("temp_frame") -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_roundtrip(conn, request, test_frame1): if conn == "sqlite_str": @@ -2292,7 +2221,6 @@ def test_roundtrip(conn, request, test_frame1): tm.assert_frame_equal(result, test_frame1) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable_iris) def test_execute_sql(conn, request): conn = request.getfixturevalue(conn) @@ -2302,7 +2230,6 @@ def test_execute_sql(conn, request): tm.equalContents(row, [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_sqlalchemy_read_table(conn, request): conn = request.getfixturevalue(conn) @@ -2310,7 +2237,6 @@ def test_sqlalchemy_read_table(conn, request): check_iris_frame(iris_frame) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_sqlalchemy_read_table_columns(conn, request): conn = request.getfixturevalue(conn) @@ -2320,7 +2246,6 @@ def test_sqlalchemy_read_table_columns(conn, request): tm.equalContents(iris_frame.columns.values, ["SepalLength", "SepalLength"]) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_read_table_absent_raises(conn, request): conn = request.getfixturevalue(conn) @@ -2329,7 +2254,6 @@ def test_read_table_absent_raises(conn, request): sql.read_sql_table("this_doesnt_exist", con=conn) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_sqlalchemy_default_type_conversion(conn, request): conn_name = conn @@ -2353,7 +2277,6 @@ def test_sqlalchemy_default_type_conversion(conn, request): assert issubclass(df.BoolColWithNull.dtype.type, object) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_bigint(conn, request): # int64 should be converted to BigInteger, GH7433 @@ -2365,7 +2288,6 @@ def test_bigint(conn, request): tm.assert_frame_equal(df, result) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_default_date_load(conn, request): conn_name = conn @@ -2382,7 +2304,6 @@ def test_default_date_load(conn, request): assert issubclass(df.DateCol.dtype.type, np.datetime64) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_datetime_with_timezone(conn, request): # edge case that converts postgresql datetime with time zone types @@ -2461,7 +2382,6 @@ def check(col): check(df.DateColWithTz) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_datetime_with_timezone_roundtrip(conn, request): conn_name = conn @@ -2493,7 +2413,6 @@ def test_datetime_with_timezone_roundtrip(conn, request): tm.assert_frame_equal(result, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_out_of_bounds_datetime(conn, request): # GH 26761 @@ -2505,7 +2424,6 @@ def test_out_of_bounds_datetime(conn, request): tm.assert_frame_equal(result, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_naive_datetimeindex_roundtrip(conn, request): # GH 23510 @@ -2519,7 +2437,6 @@ def test_naive_datetimeindex_roundtrip(conn, request): tm.assert_frame_equal(result, expected, check_names=False) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable_iris) def test_date_parsing(conn, request): # No Parsing @@ -2552,7 +2469,6 @@ def test_date_parsing(conn, request): assert issubclass(df.IntDateCol.dtype.type, np.datetime64) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_datetime(conn, request): conn_name = conn @@ -2578,7 +2494,6 @@ def test_datetime(conn, request): tm.assert_frame_equal(result, df) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_datetime_NaT(conn, request): conn_name = conn @@ -2603,7 +2518,6 @@ def test_datetime_NaT(conn, request): tm.assert_frame_equal(result, df) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_datetime_date(conn, request): # test support for datetime.date @@ -2617,7 +2531,6 @@ def test_datetime_date(conn, request): tm.assert_series_equal(result, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_datetime_time(conn, request, sqlite_buildin): # test support for datetime.time @@ -2645,7 +2558,6 @@ def test_datetime_time(conn, request, sqlite_buildin): tm.assert_frame_equal(df, res) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_mixed_dtype_insert(conn, request): # see GH6509 @@ -2661,7 +2573,6 @@ def test_mixed_dtype_insert(conn, request): tm.assert_frame_equal(df, df2, check_dtype=False, check_exact=True) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_nan_numeric(conn, request): # NaNs in numeric float column @@ -2678,7 +2589,6 @@ def test_nan_numeric(conn, request): tm.assert_frame_equal(result, df) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_nan_fullcolumn(conn, request): # full NaN column (numeric float column) @@ -2697,7 +2607,6 @@ def test_nan_fullcolumn(conn, request): tm.assert_frame_equal(result, df) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_nan_string(conn, request): # NaNs in string column @@ -2717,7 +2626,6 @@ def test_nan_string(conn, request): tm.assert_frame_equal(result, df) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_to_sql_save_index(conn, request): if "engine" in conn: @@ -2754,7 +2662,6 @@ def test_to_sql_save_index(conn, request): assert ix_cols == [["A"]] -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_transactions(conn, request): conn_name = conn @@ -2771,7 +2678,6 @@ def test_transactions(conn, request): trans.execute(stmt) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_transaction_rollback(conn, request): if "engine" in conn: @@ -2815,7 +2721,6 @@ class DummyException(Exception): assert len(res2) == 1 -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_get_schema_create_table(conn, request, test_frame3): # Use a dataframe without a bool column, since MySQL converts bool to @@ -2847,7 +2752,6 @@ def test_get_schema_create_table(conn, request, test_frame3): tm.assert_frame_equal(returned_df, blank_test_df, check_index_type=False) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_dtype(conn, request): if conn == "sqlite_str": @@ -2890,7 +2794,6 @@ def test_dtype(conn, request): assert isinstance(sqltypeb, TEXT) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_notna_dtype(conn, request): if conn == "sqlite_str": @@ -2928,7 +2831,6 @@ def test_notna_dtype(conn, request): assert isinstance(col_dict["Float"].type, Float) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_double_precision(conn, request): if conn == "sqlite_str": @@ -2981,7 +2883,6 @@ def test_double_precision(conn, request): assert isinstance(col_dict["i64"].type, BigInteger) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_connectable_issue_example(conn, request): conn = request.getfixturevalue(conn) @@ -3018,7 +2919,6 @@ def main(connectable): main(conn) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) @pytest.mark.parametrize( "input", @@ -3051,7 +2951,6 @@ def test_to_sql_with_negative_npinf(conn, request, input): tm.assert_equal(df, res) -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_temporary_table(conn, request): if conn == "sqlite_str": @@ -3090,7 +2989,6 @@ class Temporary(Base): tm.assert_frame_equal(df, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_invalid_engine(conn, request, test_frame1): if conn == "sqlite_buildin": @@ -3105,7 +3003,6 @@ def test_invalid_engine(conn, request, test_frame1): @pytest.mark.skip("Couldn't get this to work?") -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_to_sql_with_sql_engine(conn, request, test_frame1): """`to_sql` with the `engine` param""" @@ -3120,7 +3017,6 @@ def test_to_sql_with_sql_engine(conn, request, test_frame1): assert num_rows == num_entries -@pytest.mark.db @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_options_sqlalchemy(conn, request, test_frame1): if "engine" in conn: @@ -3139,7 +3035,6 @@ def test_options_sqlalchemy(conn, request, test_frame1): assert num_rows == num_entries -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_options_auto(conn, request, test_frame1): if "engine" in conn: @@ -3159,7 +3054,6 @@ def test_options_auto(conn, request, test_frame1): assert num_rows == num_entries -@pytest.mark.db @pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="fails without SQLAlchemy") def test_options_get_engine(): assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine) @@ -3180,7 +3074,6 @@ def test_get_engine_auto_error_message(): # TODO(GH#36893) fill this in when we add more engines -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) @pytest.mark.parametrize("func", ["read_sql", "read_sql_query"]) def test_read_sql_dtype_backend( @@ -3218,7 +3111,6 @@ def test_read_sql_dtype_backend( tm.assert_frame_equal(result, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) @pytest.mark.parametrize("func", ["read_sql", "read_sql_table"]) def test_read_sql_dtype_backend_table( @@ -3263,7 +3155,6 @@ def test_read_sql_dtype_backend_table( tm.assert_frame_equal(result, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) @pytest.mark.parametrize("func", ["read_sql", "read_sql_table", "read_sql_query"]) def test_read_sql_invalid_dtype_backend_table(conn, request, func, dtype_backend_data): @@ -3345,7 +3236,6 @@ def func(storage, dtype_backend, conn_name): return func -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) def test_chunksize_empty_dtypes(conn, request): # GH#50245 @@ -3364,7 +3254,6 @@ def test_chunksize_empty_dtypes(conn, request): tm.assert_frame_equal(result, expected) -@pytest.mark.db @pytest.mark.parametrize("conn", all_connectable) @pytest.mark.parametrize("dtype_backend", [lib.no_default, "numpy_nullable"]) @pytest.mark.parametrize("func", ["read_sql", "read_sql_query"]) @@ -3393,7 +3282,6 @@ def test_read_sql_dtype(conn, request, func, dtype_backend): tm.assert_frame_equal(result, expected) -@pytest.mark.db def test_keyword_deprecation(sqlite_sqlalchemy_memory_engine): conn = sqlite_sqlalchemy_memory_engine # GH 54397 @@ -3408,7 +3296,6 @@ def test_keyword_deprecation(sqlite_sqlalchemy_memory_engine): df.to_sql("example", conn, None, if_exists="replace") -@pytest.mark.db def test_bigint_warning(sqlite_sqlalchemy_memory_engine): conn = sqlite_sqlalchemy_memory_engine # test no warning for BIGINT (to support int64) is raised (GH7433) @@ -3419,7 +3306,6 @@ def test_bigint_warning(sqlite_sqlalchemy_memory_engine): sql.read_sql_table("test_bigintwarning", conn) -@pytest.mark.db def test_valueerror_exception(sqlite_sqlalchemy_memory_engine): conn = sqlite_sqlalchemy_memory_engine df = DataFrame({"col1": [1, 2], "col2": [3, 4]}) @@ -3427,7 +3313,6 @@ def test_valueerror_exception(sqlite_sqlalchemy_memory_engine): df.to_sql(name="", con=conn, if_exists="replace", index=False) -@pytest.mark.db def test_row_object_is_named_tuple(sqlite_sqlalchemy_memory_engine): conn = sqlite_sqlalchemy_memory_engine # GH 40682 @@ -3467,7 +3352,6 @@ class Test(BaseModel): assert list(df.columns) == ["id", "string_column"] -@pytest.mark.db def test_read_sql_string_inference(sqlite_sqlalchemy_memory_engine): conn = sqlite_sqlalchemy_memory_engine # GH#54430 @@ -3487,7 +3371,6 @@ def test_read_sql_string_inference(sqlite_sqlalchemy_memory_engine): tm.assert_frame_equal(result, expected) -@pytest.mark.db def test_roundtripping_datetimes(sqlite_sqlalchemy_memory_engine): conn = sqlite_sqlalchemy_memory_engine # GH#54877 @@ -3497,7 +3380,6 @@ def test_roundtripping_datetimes(sqlite_sqlalchemy_memory_engine): assert result == "2020-12-31 12:00:00.000000" -@pytest.mark.db def test_psycopg2_schema_support(postgresql_psycopg2_engine): conn = postgresql_psycopg2_engine @@ -3572,7 +3454,6 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): tm.assert_frame_equal(concat([df, df], ignore_index=True), res) -@pytest.mark.db def test_self_join_date_columns(postgresql_psycopg2_engine): # GH 44421 conn = postgresql_psycopg2_engine @@ -3608,7 +3489,6 @@ def test_self_join_date_columns(postgresql_psycopg2_engine): pandasSQL.drop_table("person") -@pytest.mark.db def test_create_and_drop_table(sqlite_sqlalchemy_memory_engine): conn = sqlite_sqlalchemy_memory_engine temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) @@ -3623,7 +3503,6 @@ def test_create_and_drop_table(sqlite_sqlalchemy_memory_engine): assert not pandasSQL.has_table("drop_test_frame") -@pytest.mark.db def test_sqlite_datetime_date(sqlite_buildin): conn = sqlite_buildin df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"]) @@ -3633,7 +3512,6 @@ def test_sqlite_datetime_date(sqlite_buildin): tm.assert_frame_equal(res, df.astype(str)) -@pytest.mark.db @pytest.mark.parametrize("tz_aware", [False, True]) def test_sqlite_datetime_time(tz_aware, sqlite_buildin): conn = sqlite_buildin @@ -3661,7 +3539,6 @@ def get_sqlite_column_type(conn, table, column): raise ValueError(f"Table {table}, column {column} not found") -@pytest.mark.db def test_sqlite_test_dtype(sqlite_buildin): conn = sqlite_buildin cols = ["A", "B"] @@ -3684,7 +3561,6 @@ def test_sqlite_test_dtype(sqlite_buildin): assert get_sqlite_column_type(conn, "single_dtype_test", "B") == "STRING" -@pytest.mark.db def test_sqlite_notna_dtype(sqlite_buildin): conn = sqlite_buildin cols = { @@ -3704,7 +3580,6 @@ def test_sqlite_notna_dtype(sqlite_buildin): assert get_sqlite_column_type(conn, tbl, "Float") == "REAL" -@pytest.mark.db def test_sqlite_illegal_names(sqlite_buildin): # For sqlite, these should work fine conn = sqlite_buildin @@ -3773,7 +3648,6 @@ def tquery(query, con=None): return None if res is None else list(res) -@pytest.mark.db def test_xsqlite_basic(sqlite_buildin): frame = tm.makeTimeDataFrame() assert sql.to_sql(frame, name="test_table", con=sqlite_buildin, index=False) == 30 @@ -3797,7 +3671,6 @@ def test_xsqlite_basic(sqlite_buildin): tm.assert_frame_equal(expected, result) -@pytest.mark.db def test_xsqlite_write_row_by_row(sqlite_buildin): frame = tm.makeTimeDataFrame() frame.iloc[0, 0] = np.nan @@ -3817,7 +3690,6 @@ def test_xsqlite_write_row_by_row(sqlite_buildin): tm.assert_frame_equal(result, frame, rtol=1e-3) -@pytest.mark.db def test_xsqlite_execute(sqlite_buildin): frame = tm.makeTimeDataFrame() create_sql = sql.get_schema(frame, "test") @@ -3835,7 +3707,6 @@ def test_xsqlite_execute(sqlite_buildin): tm.assert_frame_equal(result, frame[:1]) -@pytest.mark.db def test_xsqlite_schema(sqlite_buildin): frame = tm.makeTimeDataFrame() create_sql = sql.get_schema(frame, "test") @@ -3852,7 +3723,6 @@ def test_xsqlite_schema(sqlite_buildin): cur.execute(create_sql) -@pytest.mark.db def test_xsqlite_execute_fail(sqlite_buildin): create_sql = """ CREATE TABLE test @@ -3874,7 +3744,6 @@ def test_xsqlite_execute_fail(sqlite_buildin): pandas_sql.execute('INSERT INTO test VALUES("foo", "bar", 7)') -@pytest.mark.db def test_xsqlite_execute_closed_connection(): create_sql = """ CREATE TABLE test @@ -3897,13 +3766,11 @@ def test_xsqlite_execute_closed_connection(): tquery("select * from test", con=conn) -@pytest.mark.db def test_xsqlite_keyword_as_column_names(sqlite_buildin): df = DataFrame({"From": np.ones(5)}) assert sql.to_sql(df, con=sqlite_buildin, name="testkeywords", index=False) == 5 -@pytest.mark.db def test_xsqlite_onecolumn_of_integer(sqlite_buildin): # GH 3628 # a column_of_integers dataframe should transfer well to sql @@ -3920,7 +3787,6 @@ def test_xsqlite_onecolumn_of_integer(sqlite_buildin): tm.assert_frame_equal(result, mono_df) -@pytest.mark.db def test_xsqlite_if_exists(sqlite_buildin): df_if_exists_1 = DataFrame({"col1": [1, 2], "col2": ["A", "B"]}) df_if_exists_2 = DataFrame({"col1": [3, 4, 5], "col2": ["C", "D", "E"]}) From d968b90cc88bb219dfcfdfae751d58feab93ebff Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 11 Sep 2023 14:24:02 -0400 Subject: [PATCH 24/38] try single_cpu --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index c84bf8b41255a..23b651c28f78f 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -77,7 +77,7 @@ SQLALCHEMY_INSTALLED = False -pytestmark = [pytest.mark.db] +pytestmark = [pytest.mark.db, pytest.mark.single_cpu] @pytest.fixture From aba8d35003da65587cddf0708291df3190eba544 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 11 Sep 2023 19:39:00 -0400 Subject: [PATCH 25/38] skip more engine tests that can hang --- pandas/tests/io/test_sql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 23b651c28f78f..acb52a36c82c3 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2223,6 +2223,8 @@ def test_roundtrip(conn, request, test_frame1): @pytest.mark.parametrize("conn", all_connectable_iris) def test_execute_sql(conn, request): + if "engine" in conn: + pytest.skip(reason="hangs forever in CI with engine") conn = request.getfixturevalue(conn) pandasSQL = pandasSQL_builder(conn) iris_results = pandasSQL.execute("SELECT * FROM iris") From 8327cc6b17e388941a0f245f339708e44bdb2bc7 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 12 Sep 2023 15:12:39 -0400 Subject: [PATCH 26/38] try no pandasSQL without transaction --- pandas/tests/io/test_sql.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index acb52a36c82c3..de84129408e8a 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2666,6 +2666,9 @@ def test_to_sql_save_index(conn, request): @pytest.mark.parametrize("conn", all_connectable) def test_transactions(conn, request): + if "engine" in conn: + pytest.skip(reason="hangs forever in CI with engine") + conn_name = conn conn = request.getfixturevalue(conn) @@ -2997,6 +3000,9 @@ def test_invalid_engine(conn, request, test_frame1): request.node.add_marker( pytest.mark.xfail(reason="SQLiteDatabase does not raise for bad engine") ) + if "engine" in conn: + pytest.skip(reason="hangs forever in CI with engine") + conn = request.getfixturevalue(conn) msg = "engine must be one of 'auto', 'sqlalchemy'" pandasSQL = pandasSQL_builder(conn) @@ -3008,6 +3014,9 @@ def test_invalid_engine(conn, request, test_frame1): @pytest.mark.parametrize("conn", all_connectable) def test_to_sql_with_sql_engine(conn, request, test_frame1): """`to_sql` with the `engine` param""" + if "engine" in conn: + pytest.skip(reason="hangs forever in CI with engine") + # mostly copied from this class's `_to_sql()` method conn = request.getfixturevalue(conn) pandasSQL = pandasSQL_builder(conn) From 43b94071f874f531f4ac3a8cf1802a9b9bbeaf6d Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 12 Sep 2023 17:29:55 -0400 Subject: [PATCH 27/38] more skip --- pandas/tests/io/test_sql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index de84129408e8a..d63f5c0981b48 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1191,6 +1191,8 @@ def test_read_sql_iris_named_parameter(conn, request, sql_strings, flavor): def test_read_sql_iris_no_parameter_with_percent(conn, request, sql_strings, flavor): if "mysql" in conn or "postgresql" in conn: request.node.add_marker(pytest.mark.xfail(reason="broken test")) + if "engine" in conn: + pytest.skip(reason="can hang forever with engine") conn_name = conn conn = request.getfixturevalue(conn) From 420dc95779ab84ef84d26fd5e9013b8d2981ea81 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 12 Sep 2023 17:31:19 -0400 Subject: [PATCH 28/38] try verbose --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 48ef21686a26f..a2f25776129ef 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml" -PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -v -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From c07154161340f19a87e5968210339682522835a0 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 13 Sep 2023 19:52:32 -0400 Subject: [PATCH 29/38] transaction skips --- pandas/tests/io/test_sql.py | 43 ++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index d63f5c0981b48..370370690cb5f 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -540,9 +540,9 @@ def sqlite_engine(sqlite_str, iris_path, types_data): yield engine with engine.connect() as conn: with conn.begin(): - for view in get_all_views(engine): + for view in get_all_views(conn): drop_view(view, conn) - for tbl in get_all_tables(engine): + for tbl in get_all_tables(conn): drop_table(tbl, conn) engine.dispose() @@ -609,10 +609,10 @@ def sqlite_sqlalchemy_memory_engine(iris_path, types_data): yield engine with engine.connect() as conn: with conn.begin(): - for view in get_all_views(engine): - drop_view(view, engine) - for tbl in get_all_tables(engine): - drop_table(tbl, engine) + for view in get_all_views(conn): + drop_view(view, conn) + for tbl in get_all_tables(conn): + drop_table(tbl, conn) @pytest.fixture @@ -1162,28 +1162,25 @@ def func(conn_name): @pytest.mark.parametrize("conn", all_connectable_iris) def test_read_sql_iris_parameter(conn, request, sql_strings, flavor): conn_name = conn - if "engine" in conn: - pytest.skip(reason="can hang forever with engine") - conn = request.getfixturevalue(conn) query = sql_strings["read_parameters"][flavor(conn_name)] params = ("Iris-setosa", 5.1) pandasSQL = pandasSQL_builder(conn) - iris_frame = pandasSQL.read_query(query, params=params) + + with pandasSQL.run_transaction(): + iris_frame = pandasSQL.read_query(query, params=params) check_iris_frame(iris_frame) @pytest.mark.parametrize("conn", all_connectable_iris) def test_read_sql_iris_named_parameter(conn, request, sql_strings, flavor): conn_name = conn - if "engine" in conn: - pytest.skip(reason="can hang forever with engine") - conn = request.getfixturevalue(conn) query = sql_strings["read_named_parameters"][flavor(conn_name)] params = {"name": "Iris-setosa", "length": 5.1} pandasSQL = pandasSQL_builder(conn) - iris_frame = pandasSQL.read_query(query, params=params) + with pandasSQL.run_transaction(): + iris_frame = pandasSQL.read_query(query, params=params) check_iris_frame(iris_frame) @@ -1191,15 +1188,14 @@ def test_read_sql_iris_named_parameter(conn, request, sql_strings, flavor): def test_read_sql_iris_no_parameter_with_percent(conn, request, sql_strings, flavor): if "mysql" in conn or "postgresql" in conn: request.node.add_marker(pytest.mark.xfail(reason="broken test")) - if "engine" in conn: - pytest.skip(reason="can hang forever with engine") conn_name = conn conn = request.getfixturevalue(conn) query = sql_strings["read_no_parameters_with_percent"][flavor(conn_name)] pandasSQL = pandasSQL_builder(conn) - iris_frame = pandasSQL.read_query(query, params=None) + with pandasSQL.run_transaction(): + iris_frame = pandasSQL.read_query(query, params=None) check_iris_frame(iris_frame) @@ -2205,15 +2201,14 @@ def test_drop_table(conn, request): def test_roundtrip(conn, request, test_frame1): if conn == "sqlite_str": pytest.skip("sqlite_str has no inspection system") - elif "engine" in conn: - pytest.skip(reason="fails and hangs forever with engine") conn = request.getfixturevalue(conn) drop_table("test_frame_roundtrip", conn) pandasSQL = pandasSQL_builder(conn) assert pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4 - result = pandasSQL.read_query("SELECT * FROM test_frame_roundtrip") + with pandasSQL.run_transaction(): + result = pandasSQL.read_query("SELECT * FROM test_frame_roundtrip") result.set_index("level_0", inplace=True) # result.index.astype(int) @@ -2687,9 +2682,6 @@ def test_transactions(conn, request): @pytest.mark.parametrize("conn", all_connectable) def test_transaction_rollback(conn, request): - if "engine" in conn: - pytest.skip(reason="fails and hangs forever with engine") - conn = request.getfixturevalue(conn) pandasSQL = pandasSQL_builder(conn) with pandasSQL.run_transaction() as trans: @@ -2718,13 +2710,14 @@ class DummyException(Exception): except DummyException: # ignore raised exception pass - res = pandasSQL.read_query("SELECT * FROM test_trans") + with pandasSQL.run_transaction(): + res = pandasSQL.read_query("SELECT * FROM test_trans") assert len(res) == 0 # Make sure when transaction is committed, rows do get inserted with pandasSQL.run_transaction() as trans: trans.execute(ins_sql) - res2 = pandasSQL.read_query("SELECT * FROM test_trans") + res2 = pandasSQL.read_query("SELECT * FROM test_trans") assert len(res2) == 1 From 7c8c54fccbaad5763ac28c14279f6b37c080a82d Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Wed, 13 Sep 2023 19:59:02 -0400 Subject: [PATCH 30/38] remove verbose CI --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index a2f25776129ef..48ef21686a26f 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml" -PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -v -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 4f3997cc14b85b26c41106e9c843fd370d1c7e47 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 14 Sep 2023 14:29:25 -0400 Subject: [PATCH 31/38] CI verbose --- .github/workflows/unit-tests.yml | 16 ++++++++-------- ci/run_tests.sh | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index f2b426269098b..6dcc4fee72e54 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -23,7 +23,7 @@ defaults: jobs: ubuntu: runs-on: ubuntu-22.04 - timeout-minutes: 180 + timeout-minutes: 30 strategy: matrix: env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml] @@ -158,13 +158,6 @@ jobs: id: build uses: ./.github/actions/build_pandas - - name: Test (not single_cpu) - uses: ./.github/actions/run-tests - if: ${{ matrix.name != 'Pypy' }} - env: - # Set pattern to not single_cpu if not already set - PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }} - - name: Test (single_cpu) uses: ./.github/actions/run-tests env: @@ -172,6 +165,13 @@ jobs: PYTEST_WORKERS: 0 if: ${{ matrix.pattern == '' && (always() && steps.build.outcome == 'success')}} + - name: Test (not single_cpu) + uses: ./.github/actions/run-tests + if: ${{ matrix.name != 'Pypy' }} + env: + # Set pattern to not single_cpu if not already set + PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }} + macos-windows: timeout-minutes: 180 strategy: diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 48ef21686a26f..a2f25776129ef 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml" -PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -v -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From f4f5241c93595002d8031c47d6b8e49f448e48f0 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 14 Sep 2023 14:51:29 -0400 Subject: [PATCH 32/38] no more hanging --- pandas/tests/io/test_sql.py | 56 ++++++++++++++----------------------- 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 370370690cb5f..ec267e1a9dc90 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -428,7 +428,8 @@ def drop_table( conn.execute(f"DROP TABLE IF EXISTS {sql._get_valid_sqlite_name(table_name)}") conn.commit() else: - sql.SQLDatabase(conn).drop_table(table_name) + with conn.begin() as con: + sql.SQLDatabase(con).drop_table(table_name) def drop_view( @@ -443,11 +444,8 @@ def drop_view( view_name ) stmt = sqlalchemy.text(f"DROP VIEW IF EXISTS {quoted_view}") - if isinstance(conn, sqlalchemy.engine.Engine): - with conn.connect() as conn: - conn.execute(stmt) - else: - conn.execute(stmt) + with conn.begin() as con: + con.execute(stmt) @pytest.fixture @@ -469,12 +467,10 @@ def mysql_pymysql_engine(iris_path, types_data): if not insp.has_table("iris_view"): create_and_load_iris_view(engine) yield engine - with engine.connect() as conn: - with conn.begin(): - for view in get_all_views(conn): - drop_view(view, conn) - for tbl in get_all_tables(conn): - drop_table(tbl, conn) + for view in get_all_views(engine): + drop_view(view, engine) + for tbl in get_all_tables(engine): + drop_table(tbl, engine) engine.dispose() @@ -500,12 +496,10 @@ def postgresql_psycopg2_engine(iris_path, types_data): if not insp.has_table("iris_view"): create_and_load_iris_view(engine) yield engine - with engine.connect() as conn: - with conn.begin(): - for view in get_all_views(conn): - drop_view(view, conn) - for tbl in get_all_tables(conn): - drop_table(tbl, conn) + for view in get_all_views(engine): + drop_view(view, engine) + for tbl in get_all_tables(engine): + drop_table(tbl, engine) engine.dispose() @@ -538,13 +532,10 @@ def sqlite_engine(sqlite_str, iris_path, types_data): create_and_load_types(engine, types_data, "sqlite") yield engine - with engine.connect() as conn: - with conn.begin(): - for view in get_all_views(conn): - drop_view(view, conn) - for tbl in get_all_tables(conn): - drop_table(tbl, conn) - + for view in get_all_views(engine): + drop_view(view, engine) + for tbl in get_all_tables(engine): + drop_table(tbl, engine) engine.dispose() @@ -607,12 +598,10 @@ def sqlite_sqlalchemy_memory_engine(iris_path, types_data): create_and_load_types(engine, types_data, "sqlite") yield engine - with engine.connect() as conn: - with conn.begin(): - for view in get_all_views(conn): - drop_view(view, conn) - for tbl in get_all_tables(conn): - drop_table(tbl, conn) + for view in get_all_views(engine): + drop_view(view, engine) + for tbl in get_all_tables(engine): + drop_table(tbl, engine) @pytest.fixture @@ -2203,11 +2192,9 @@ def test_roundtrip(conn, request, test_frame1): pytest.skip("sqlite_str has no inspection system") conn = request.getfixturevalue(conn) - drop_table("test_frame_roundtrip", conn) - pandasSQL = pandasSQL_builder(conn) - assert pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4 with pandasSQL.run_transaction(): + assert pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4 result = pandasSQL.read_query("SELECT * FROM test_frame_roundtrip") result.set_index("level_0", inplace=True) @@ -2740,7 +2727,6 @@ def test_get_schema_create_table(conn, request, test_frame3): create_sql = sql.get_schema(test_frame3, tbl, con=conn) blank_test_df = test_frame3.iloc[:0] - drop_table(tbl, conn) create_sql = text(create_sql) if isinstance(conn, Engine): with conn.connect() as newcon: From 3b13e1e18ccab9d841f68cfe0e93454378b754c4 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 14 Sep 2023 16:20:34 -0400 Subject: [PATCH 33/38] reverted CI files --- .github/workflows/unit-tests.yml | 16 ++++++++-------- ci/run_tests.sh | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6dcc4fee72e54..f2b426269098b 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -23,7 +23,7 @@ defaults: jobs: ubuntu: runs-on: ubuntu-22.04 - timeout-minutes: 30 + timeout-minutes: 180 strategy: matrix: env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml] @@ -158,13 +158,6 @@ jobs: id: build uses: ./.github/actions/build_pandas - - name: Test (single_cpu) - uses: ./.github/actions/run-tests - env: - PATTERN: 'single_cpu' - PYTEST_WORKERS: 0 - if: ${{ matrix.pattern == '' && (always() && steps.build.outcome == 'success')}} - - name: Test (not single_cpu) uses: ./.github/actions/run-tests if: ${{ matrix.name != 'Pypy' }} @@ -172,6 +165,13 @@ jobs: # Set pattern to not single_cpu if not already set PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }} + - name: Test (single_cpu) + uses: ./.github/actions/run-tests + env: + PATTERN: 'single_cpu' + PYTEST_WORKERS: 0 + if: ${{ matrix.pattern == '' && (always() && steps.build.outcome == 'success')}} + macos-windows: timeout-minutes: 180 strategy: diff --git a/ci/run_tests.sh b/ci/run_tests.sh index a2f25776129ef..48ef21686a26f 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml" -PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -v -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 8978d61b2598b074a10aa68057e08af33cefcf50 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Thu, 14 Sep 2023 19:17:33 -0400 Subject: [PATCH 34/38] type ignore --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index ec267e1a9dc90..7e801f191dc76 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -445,7 +445,7 @@ def drop_view( ) stmt = sqlalchemy.text(f"DROP VIEW IF EXISTS {quoted_view}") with conn.begin() as con: - con.execute(stmt) + con.execute(stmt) # type: ignore[union-attr] @pytest.fixture From 6cf13d689bfad71abd3368b77639b44c1e99c5f8 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Sep 2023 14:03:29 -0400 Subject: [PATCH 35/38] cleanup skips --- pandas/tests/io/test_sql.py | 58 ++++++++++++++----------------------- 1 file changed, 21 insertions(+), 37 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 7e801f191dc76..f56290c3e11cf 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2164,8 +2164,6 @@ def test_create_table(conn, request): def test_drop_table(conn, request): if conn == "sqlite_str": pytest.skip("sqlite_str has no inspection system") - elif "engine" in conn: - pytest.skip(reason="fails and hangs forever with engine") conn = request.getfixturevalue(conn) @@ -2173,12 +2171,14 @@ def test_drop_table(conn, request): temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) pandasSQL = sql.SQLDatabase(conn) - assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 + with pandasSQL.run_transaction(): + assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4 insp = inspect(conn) assert insp.has_table("temp_frame") - pandasSQL.drop_table("temp_frame") + with pandasSQL.run_transaction(): + pandasSQL.drop_table("temp_frame") try: insp.clear_cache() # needed with SQLAlchemy 2.0, unavailable prior except AttributeError: @@ -2207,11 +2207,10 @@ def test_roundtrip(conn, request, test_frame1): @pytest.mark.parametrize("conn", all_connectable_iris) def test_execute_sql(conn, request): - if "engine" in conn: - pytest.skip(reason="hangs forever in CI with engine") conn = request.getfixturevalue(conn) pandasSQL = pandasSQL_builder(conn) - iris_results = pandasSQL.execute("SELECT * FROM iris") + with pandasSQL.run_transaction(): + iris_results = pandasSQL.execute("SELECT * FROM iris") row = iris_results.fetchone() tm.equalContents(row, [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]) @@ -2614,9 +2613,6 @@ def test_nan_string(conn, request): @pytest.mark.parametrize("conn", all_connectable) def test_to_sql_save_index(conn, request): - if "engine" in conn: - pytest.skip(reason="fails and hangs forever with engine") - conn_name = conn conn = request.getfixturevalue(conn) df = DataFrame.from_records( @@ -2625,7 +2621,8 @@ def test_to_sql_save_index(conn, request): pandasSQL = pandasSQL_builder(conn) tbl_name = "test_to_sql_saves_index" - assert pandasSQL.to_sql(df, tbl_name) == 2 + with pandasSQL.run_transaction(): + assert pandasSQL.to_sql(df, tbl_name) == 2 if conn_name in {"sqlite_buildin", "sqlite_str"}: ixs = sql.read_sql_query( @@ -2650,9 +2647,6 @@ def test_to_sql_save_index(conn, request): @pytest.mark.parametrize("conn", all_connectable) def test_transactions(conn, request): - if "engine" in conn: - pytest.skip(reason="hangs forever in CI with engine") - conn_name = conn conn = request.getfixturevalue(conn) @@ -2981,8 +2975,6 @@ def test_invalid_engine(conn, request, test_frame1): request.node.add_marker( pytest.mark.xfail(reason="SQLiteDatabase does not raise for bad engine") ) - if "engine" in conn: - pytest.skip(reason="hangs forever in CI with engine") conn = request.getfixturevalue(conn) msg = "engine must be one of 'auto', 'sqlalchemy'" @@ -2991,18 +2983,15 @@ def test_invalid_engine(conn, request, test_frame1): pandasSQL.to_sql(test_frame1, "test_frame1", engine="bad_engine") -@pytest.mark.skip("Couldn't get this to work?") @pytest.mark.parametrize("conn", all_connectable) def test_to_sql_with_sql_engine(conn, request, test_frame1): """`to_sql` with the `engine` param""" - if "engine" in conn: - pytest.skip(reason="hangs forever in CI with engine") - # mostly copied from this class's `_to_sql()` method conn = request.getfixturevalue(conn) pandasSQL = pandasSQL_builder(conn) - assert pandasSQL.to_sql(test_frame1, "test_frame1", engine="auto") == 4 - assert pandasSQL.has_table("test_frame1") + with pandasSQL.run_transaction(): + assert pandasSQL.to_sql(test_frame1, "test_frame1", engine="auto") == 4 + assert pandasSQL.has_table("test_frame1") num_entries = len(test_frame1) num_rows = count_rows(conn, "test_frame1") @@ -3011,17 +3000,14 @@ def test_to_sql_with_sql_engine(conn, request, test_frame1): @pytest.mark.parametrize("conn", sqlalchemy_connectable) def test_options_sqlalchemy(conn, request, test_frame1): - if "engine" in conn: - pytest.skip("fails and hangs forever with engine") - elif conn == "sqlite_str": - pytest.skip("may hang on old versions of sqlite") # use the set option conn = request.getfixturevalue(conn) with pd.option_context("io.sql.engine", "sqlalchemy"): pandasSQL = pandasSQL_builder(conn) - assert pandasSQL.to_sql(test_frame1, "test_frame1") == 4 + with pandasSQL.run_transaction(): + assert pandasSQL.to_sql(test_frame1, "test_frame1") == 4 + assert pandasSQL.has_table("test_frame1") - assert pandasSQL.has_table("test_frame1") num_entries = len(test_frame1) num_rows = count_rows(conn, "test_frame1") assert num_rows == num_entries @@ -3029,18 +3015,14 @@ def test_options_sqlalchemy(conn, request, test_frame1): @pytest.mark.parametrize("conn", all_connectable) def test_options_auto(conn, request, test_frame1): - if "engine" in conn: - pytest.skip("fails and hangs forever with engine") - elif conn == "sqlite_str": - pytest.skip("may hang on old versions of sqlite") - # use the set option conn = request.getfixturevalue(conn) with pd.option_context("io.sql.engine", "auto"): pandasSQL = pandasSQL_builder(conn) - assert pandasSQL.to_sql(test_frame1, "test_frame1") == 4 + with pandasSQL.run_transaction(): + assert pandasSQL.to_sql(test_frame1, "test_frame1") == 4 + assert pandasSQL.has_table("test_frame1") - assert pandasSQL.has_table("test_frame1") num_entries = len(test_frame1) num_rows = count_rows(conn, "test_frame1") assert num_rows == num_entries @@ -3486,11 +3468,13 @@ def test_create_and_drop_table(sqlite_sqlalchemy_memory_engine): temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]}) pandasSQL = sql.SQLDatabase(conn) - assert pandasSQL.to_sql(temp_frame, "drop_test_frame") == 4 + with pandasSQL.run_transaction(): + assert pandasSQL.to_sql(temp_frame, "drop_test_frame") == 4 assert pandasSQL.has_table("drop_test_frame") - pandasSQL.drop_table("drop_test_frame") + with pandasSQL.run_transaction(): + pandasSQL.drop_table("drop_test_frame") assert not pandasSQL.has_table("drop_test_frame") From 424ada1585566b27bdf065f2c7fa377201a6b4b5 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Sep 2023 14:24:00 -0400 Subject: [PATCH 36/38] remove marks --- pandas/tests/io/test_sql.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index f56290c3e11cf..d0af01b3f5364 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -77,9 +77,6 @@ SQLALCHEMY_INSTALLED = False -pytestmark = [pytest.mark.db, pytest.mark.single_cpu] - - @pytest.fixture def sql_strings(): return { From 359641d7dd0df5b6e5857f9c1de1dbdd5ad099eb Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Sep 2023 14:34:28 -0400 Subject: [PATCH 37/38] mark fixtures --- pandas/tests/io/test_sql.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index d0af01b3f5364..17057110cd1b8 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -614,26 +614,26 @@ def sqlite_buildin_iris(sqlite_buildin, iris_path, types_data): mysql_connectable = [ - "mysql_pymysql_engine", - "mysql_pymysql_conn", + pytest.param("mysql_pymysql_engine", marks=pytest.mark.db), + pytest.param("mysql_pymysql_conn", marks=pytest.mark.db), ] postgresql_connectable = [ - "postgresql_psycopg2_engine", - "postgresql_psycopg2_conn", + pytest.param("postgresql_psycopg2_engine", marks=pytest.mark.db), + pytest.param("postgresql_psycopg2_conn", marks=pytest.mark.db), ] sqlite_connectable = [ - "sqlite_engine", - "sqlite_conn", - "sqlite_str", + pytest.param("sqlite_engine", marks=pytest.mark.db), + pytest.param("sqlite_conn", marks=pytest.mark.db), + pytest.param("sqlite_str", marks=pytest.mark.db), ] sqlite_iris_connectable = [ - "sqlite_iris_engine", - "sqlite_iris_conn", - "sqlite_iris_str", + pytest.param("sqlite_iris_engine", marks=pytest.mark.db), + pytest.param("sqlite_iris_conn", marks=pytest.mark.db), + pytest.param("sqlite_iris_str", marks=pytest.mark.db), ] sqlalchemy_connectable = mysql_connectable + postgresql_connectable + sqlite_connectable From 587e53da5056a7eb783f33efac750af359c4f0dd Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 18 Sep 2023 15:48:08 -0400 Subject: [PATCH 38/38] mark postgres fixtures --- pandas/tests/io/test_sql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 17057110cd1b8..f015c9efe7122 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -3351,6 +3351,7 @@ def test_roundtripping_datetimes(sqlite_sqlalchemy_memory_engine): assert result == "2020-12-31 12:00:00.000000" +@pytest.mark.db def test_psycopg2_schema_support(postgresql_psycopg2_engine): conn = postgresql_psycopg2_engine @@ -3425,6 +3426,7 @@ def test_psycopg2_schema_support(postgresql_psycopg2_engine): tm.assert_frame_equal(concat([df, df], ignore_index=True), res) +@pytest.mark.db def test_self_join_date_columns(postgresql_psycopg2_engine): # GH 44421 conn = postgresql_psycopg2_engine