From 4a666a8acf2a5cdc5bd1462fd858bec5cf71bf30 Mon Sep 17 00:00:00 2001 From: Grigi Date: Tue, 2 Oct 2018 13:01:25 +0200 Subject: [PATCH 1/9] _prepare_insert_columns and _prepare_insert_values now do what you'd expect --- tortoise/backends/asyncpg/executor.py | 8 ++++---- tortoise/backends/base/executor.py | 12 ++++-------- tortoise/backends/mysql/executor.py | 8 ++++---- tortoise/backends/sqlite/executor.py | 10 +++++----- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/tortoise/backends/asyncpg/executor.py b/tortoise/backends/asyncpg/executor.py index bbae31d80..632646882 100644 --- a/tortoise/backends/asyncpg/executor.py +++ b/tortoise/backends/asyncpg/executor.py @@ -6,17 +6,17 @@ class AsyncpgExecutor(BaseExecutor): async def execute_insert(self, instance): self.connection = await self.db.get_single_connection() - regular_columns = self._prepare_insert_columns() - columns, values = self._prepare_insert_values( + regular_columns, columns = self._prepare_insert_columns() + values = self._prepare_insert_values( instance=instance, regular_columns=regular_columns, ) - query = ( + query = str( self.connection.query_class.into(Table(self.model._meta.table)).columns(*columns) .insert(*values).returning('id') ) - result = await self.connection.execute_query(str(query)) + result = await self.connection.execute_query(query) instance.id = result[0][0] await self.db.release_single_connection(self.connection) self.connection = None diff --git a/tortoise/backends/base/executor.py b/tortoise/backends/base/executor.py index 2810c44a6..46bd32682 100644 --- a/tortoise/backends/base/executor.py +++ b/tortoise/backends/base/executor.py @@ -37,21 +37,17 @@ def _prepare_insert_columns(self): field_object = self.model._meta.fields_map[column] if not field_object.generated: regular_columns.append(column) - return regular_columns + result_columns = [self.model._meta.fields_db_projection[c] for c in regular_columns] + return regular_columns, result_columns def _field_to_db(self, field_object, attr, instance): if field_object.__class__ in self.TO_DB_OVERRIDE: return self.TO_DB_OVERRIDE[field_object.__class__](field_object, attr, instance) return field_object.to_db_value(attr, instance) - def _get_prepared_value(self, instance, column): - return self._field_to_db(self.model._meta.fields_map[column], getattr(instance, column), - instance) - def _prepare_insert_values(self, instance, regular_columns): - values = [self._get_prepared_value(instance, column) for column in regular_columns] - result_columns = [self.model._meta.fields_db_projection[c] for c in regular_columns] - return result_columns, values + return [self._field_to_db(self.model._meta.fields_map[column], getattr(instance, column), + instance) for column in regular_columns] async def execute_insert(self, instance): # Insert should implement returning new id to saved object diff --git a/tortoise/backends/mysql/executor.py b/tortoise/backends/mysql/executor.py index 3bea8080c..37eb389e8 100644 --- a/tortoise/backends/mysql/executor.py +++ b/tortoise/backends/mysql/executor.py @@ -49,18 +49,18 @@ def mysql_insensitive_ends_with(field, value): class MySQLExecutor(BaseExecutor): async def execute_insert(self, instance): self.connection = await self.db.get_single_connection() - regular_columns = self._prepare_insert_columns() - columns, values = self._prepare_insert_values( + regular_columns, columns = self._prepare_insert_columns() + values = self._prepare_insert_values( instance=instance, regular_columns=regular_columns, ) - query = ( + query = str( MySQLQuery.into(Table(self.model._meta.table)).columns(*columns) .insert(*values) ) - instance.id = await self.connection.execute_query(str(query), get_inserted_id=True) + instance.id = await self.connection.execute_query(query, get_inserted_id=True) await self.db.release_single_connection(self.connection) self.connection = None return instance diff --git a/tortoise/backends/sqlite/executor.py b/tortoise/backends/sqlite/executor.py index c676a882c..81d4e8b9b 100644 --- a/tortoise/backends/sqlite/executor.py +++ b/tortoise/backends/sqlite/executor.py @@ -30,17 +30,17 @@ class SqliteExecutor(BaseExecutor): async def execute_insert(self, instance): self.connection = await self.db.get_single_connection() - regular_columns = self._prepare_insert_columns() - columns, values = self._prepare_insert_values( + regular_columns, columns = self._prepare_insert_columns() + values = self._prepare_insert_values( instance=instance, regular_columns=regular_columns, ) - - query = ( + query = str( self.connection.query_class.into(Table(self.model._meta.table)).columns(*columns) .insert(*values) ) - result = await self.connection.execute_query(str(query), get_inserted_id=True) + + result = await self.connection.execute_query(query, get_inserted_id=True) instance.id = result[0] await self.db.release_single_connection(self.connection) self.connection = None From 084829cd1e60cfecc2fc90e7094685c550e3d946 Mon Sep 17 00:00:00 2001 From: Grigi Date: Tue, 2 Oct 2018 22:30:15 +0200 Subject: [PATCH 2/9] Split execute_query into execute_insert and type DB Client classes --- tortoise/backends/asyncpg/client.py | 81 +++++++++++++----------- tortoise/backends/asyncpg/executor.py | 3 +- tortoise/backends/base/client.py | 40 ++++++------ tortoise/backends/mysql/client.py | 89 ++++++++++++++------------ tortoise/backends/mysql/executor.py | 2 +- tortoise/backends/sqlite/client.py | 91 ++++++++++++++------------- tortoise/backends/sqlite/executor.py | 3 +- 7 files changed, 168 insertions(+), 141 deletions(-) diff --git a/tortoise/backends/asyncpg/client.py b/tortoise/backends/asyncpg/client.py index 4a3d3c36f..46ab68d16 100644 --- a/tortoise/backends/asyncpg/client.py +++ b/tortoise/backends/asyncpg/client.py @@ -1,4 +1,6 @@ import logging +from functools import wraps +from typing import List, SupportsInt, Optional # noqa import asyncpg from pypika import PostgreSQLQuery @@ -12,13 +14,27 @@ from tortoise.transactions import current_transaction_map +def translate_exceptions(func): + @wraps(func) + async def wrapped(self, query): + self.log.debug(query) + try: + return await func(self, query) + except asyncpg.exceptions.SyntaxOrAccessError as exc: + raise OperationalError(exc) + except asyncpg.exceptions.IntegrityConstraintViolationError as exc: + raise IntegrityError(exc) + return wrapped + + class AsyncpgDBClient(BaseDBAsyncClient): DSN_TEMPLATE = 'postgres://{user}:{password}@{host}:{port}/{database}' query_class = PostgreSQLQuery executor_class = AsyncpgExecutor schema_generator = AsyncpgSchemaGenerator - def __init__(self, user, password, database, host, port, **kwargs): + def __init__(self, user: str, password: str, database: str, host: str, port: SupportsInt, + **kwargs) -> None: super().__init__(**kwargs) self.user = user @@ -34,14 +50,14 @@ def __init__(self, user, password, database, host, port, **kwargs): port=self.port, database=self.database ) - self._db_pool = None - self._connection = None + self._db_pool = None # Type: Optional[asyncpg.pool.Pool] + self._connection = None # Type: Optional[asyncpg.Connection] self._transaction_class = type( 'TransactionWrapper', (TransactionWrapper, self.__class__), {} ) - async def create_connection(self): + async def create_connection(self) -> None: try: if not self.single_connection: self._db_pool = await asyncpg.create_pool(self.dsn) @@ -56,13 +72,13 @@ async def create_connection(self): self.database )) - async def close(self): - if not self.single_connection: + async def close(self) -> None: + if self._db_pool: await self._db_pool.close() - else: + if self._connection: await self._connection.close() - async def db_create(self): + async def db_create(self) -> None: single_connection = self.single_connection self.single_connection = True self._connection = await asyncpg.connect(self.DSN_TEMPLATE.format( @@ -75,10 +91,10 @@ async def db_create(self): await self.execute_script( 'CREATE DATABASE "{}" OWNER "{}"'.format(self.database, self.user) ) - await self._connection.close() + await self._connection.close() # type: ignore self.single_connection = single_connection - async def db_delete(self): + async def db_delete(self) -> None: single_connection = self.single_connection self.single_connection = True self._connection = await asyncpg.connect(self.DSN_TEMPLATE.format( @@ -92,30 +108,35 @@ async def db_delete(self): await self.execute_script('DROP DATABASE "{}"'.format(self.database)) except asyncpg.InvalidCatalogNameError: pass - await self._connection.close() + await self._connection.close() # type: ignore self.single_connection = single_connection - def acquire_connection(self): + def acquire_connection(self) -> ConnectionWrapper: if not self.single_connection: - return self._db_pool.acquire() + return self._db_pool.acquire() # type: ignore else: return ConnectionWrapper(self._connection) - def _in_transaction(self): + def _in_transaction(self) -> 'TransactionWrapper': if self.single_connection: return self._transaction_class(self.connection_name, connection=self._connection) else: return self._transaction_class(self.connection_name, pool=self._db_pool) - async def execute_query(self, query, get_inserted_id=False): - try: - async with self.acquire_connection() as connection: - self.log.debug(query) - return await connection.fetch(query) - except asyncpg.exceptions.SyntaxOrAccessError as exc: - raise OperationalError(exc) - except asyncpg.exceptions.IntegrityConstraintViolationError as exc: - raise IntegrityError(exc) + @translate_exceptions + async def execute_insert(self, query: str) -> int: + async with self.acquire_connection() as connection: + return (await connection.fetch(query))[0][0] + + @translate_exceptions + async def execute_query(self, query: str) -> List[dict]: + async with self.acquire_connection() as connection: + return await connection.fetch(query) + + @translate_exceptions + async def execute_script(self, query: str) -> None: + async with self.acquire_connection() as connection: + await connection.execute(query) async def get_single_connection(self): if self.single_connection: @@ -128,19 +149,9 @@ async def release_single_connection(self, single_connection): if not self.single_connection: await self._db_pool.release(single_connection.connection) - async def execute_script(self, script): - try: - async with self.acquire_connection() as connection: - self.log.debug(script) - await connection.execute(script) - except asyncpg.exceptions.SyntaxOrAccessError as exc: - raise OperationalError(exc) - except asyncpg.exceptions.IntegrityConstraintViolationError as exc: - raise IntegrityError(exc) - class TransactionWrapper(AsyncpgDBClient, BaseTransactionWrapper): - def __init__(self, connection_name, pool=None, connection=None): + def __init__(self, connection_name: str, pool=None, connection=None) -> None: if pool and connection: raise ConfigurationError('You must pass either connection or pool') self._connection = connection @@ -155,7 +166,7 @@ def __init__(self, connection_name, pool=None, connection=None): self.connection_name = connection_name self.transaction = None - def acquire_connection(self): + def acquire_connection(self) -> ConnectionWrapper: return ConnectionWrapper(self._connection) async def _get_connection(self): diff --git a/tortoise/backends/asyncpg/executor.py b/tortoise/backends/asyncpg/executor.py index 632646882..8c8517b39 100644 --- a/tortoise/backends/asyncpg/executor.py +++ b/tortoise/backends/asyncpg/executor.py @@ -16,8 +16,7 @@ async def execute_insert(self, instance): self.connection.query_class.into(Table(self.model._meta.table)).columns(*columns) .insert(*values).returning('id') ) - result = await self.connection.execute_query(query) - instance.id = result[0][0] + instance.id = await self.connection.execute_insert(query) await self.db.release_single_connection(self.connection) self.connection = None return instance diff --git a/tortoise/backends/base/client.py b/tortoise/backends/base/client.py index 7545238ff..3e6df629e 100644 --- a/tortoise/backends/base/client.py +++ b/tortoise/backends/base/client.py @@ -1,4 +1,5 @@ import logging +from typing import Sequence from pypika import Query @@ -11,7 +12,7 @@ class BaseDBAsyncClient: executor_class = BaseExecutor schema_generator = BaseSchemaGenerator - def __init__(self, connection_name, single_connection=True, **kwargs): + def __init__(self, connection_name: str, single_connection: bool=True, **kwargs) -> None: self.log = logging.getLogger('db_client') self.single_connection = single_connection self.connection_name = connection_name @@ -19,34 +20,37 @@ def __init__(self, connection_name, single_connection=True, **kwargs): 'SingleConnectionWrapper', (SingleConnectionWrapper, self.__class__), {} ) - async def create_connection(self): + async def create_connection(self) -> None: raise NotImplementedError() # pragma: nocoverage - async def close(self): + async def close(self) -> None: raise NotImplementedError() # pragma: nocoverage - async def db_create(self): + async def db_create(self) -> None: raise NotImplementedError() # pragma: nocoverage - async def db_delete(self): + async def db_delete(self) -> None: raise NotImplementedError() # pragma: nocoverage - def acquire_connection(self): + def acquire_connection(self) -> 'ConnectionWrapper': raise NotImplementedError() # pragma: nocoverage - def _in_transaction(self): + def _in_transaction(self) -> 'BaseTransactionWrapper': raise NotImplementedError() # pragma: nocoverage - async def execute_query(self, query, get_inserted_id=False): + async def execute_insert(self, query: str) -> int: raise NotImplementedError() # pragma: nocoverage - async def execute_script(self, script): + async def execute_query(self, query: str) -> Sequence[dict]: raise NotImplementedError() # pragma: nocoverage - async def get_single_connection(self): + async def execute_script(self, query: str) -> None: raise NotImplementedError() # pragma: nocoverage - async def release_single_connection(self, single_connection): + async def get_single_connection(self) -> 'BaseDBAsyncClient': + raise NotImplementedError() # pragma: nocoverage + + async def release_single_connection(self, single_connection: 'BaseDBAsyncClient') -> None: raise NotImplementedError() # pragma: nocoverage @@ -64,22 +68,22 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): class SingleConnectionWrapper(BaseDBAsyncClient): # pylint: disable=W0223,W0231 - def __init__(self, connection_name, connection, closing_callback=None): + def __init__(self, connection_name: str, connection, closing_callback=None) -> None: self.log = logging.getLogger('db_client') self.connection_name = connection_name self.connection = connection self.single_connection = True self.closing_callback = closing_callback - def acquire_connection(self): + def acquire_connection(self) -> ConnectionWrapper: return ConnectionWrapper(self.connection) - async def get_single_connection(self): + async def get_single_connection(self) -> 'SingleConnectionWrapper': # Real class object is generated in runtime, so we use __class__ reference # instead of using SingleConnectionWrapper directly return self.__class__(self.connection_name, self.connection, self) - async def release_single_connection(self, single_connection): + async def release_single_connection(self, single_connection: 'BaseDBAsyncClient') -> None: return async def __aenter__(self): @@ -91,13 +95,13 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): class BaseTransactionWrapper: - async def start(self): + async def start(self) -> None: raise NotImplementedError() # pragma: nocoverage - async def rollback(self): + async def rollback(self) -> None: raise NotImplementedError() # pragma: nocoverage - async def commit(self): + async def commit(self) -> None: raise NotImplementedError() # pragma: nocoverage async def __aenter__(self): diff --git a/tortoise/backends/mysql/client.py b/tortoise/backends/mysql/client.py index b59cff463..2d0cc607d 100644 --- a/tortoise/backends/mysql/client.py +++ b/tortoise/backends/mysql/client.py @@ -1,4 +1,6 @@ import logging +from functools import wraps +from typing import List, SupportsInt, Optional # noqa import aiomysql import pymysql @@ -13,12 +15,28 @@ from tortoise.transactions import current_transaction_map +def translate_exceptions(func): + @wraps(func) + async def wrapped(self, query): + self.log.debug(query) + try: + return await func(self, query) + except (pymysql.err.OperationalError, pymysql.err.ProgrammingError, + pymysql.err.DataError, pymysql.err.InternalError, + pymysql.err.NotSupportedError) as exc: + raise OperationalError(exc) + except pymysql.err.IntegrityError as exc: + raise IntegrityError(exc) + return wrapped + + class MySQLClient(BaseDBAsyncClient): query_class = MySQLQuery executor_class = MySQLExecutor schema_generator = MySQLSchemaGenerator - def __init__(self, user, password, database, host, port, **kwargs): + def __init__(self, user: str, password: str, database: str, host: str, port: SupportsInt, + **kwargs) -> None: super().__init__(**kwargs) self.user = user @@ -34,14 +52,14 @@ def __init__(self, user, password, database, host, port, **kwargs): 'password': self.password, } - self._db_pool = None - self._connection = None + self._db_pool = None # Type: Optional[aiomysql.Pool] + self._connection = None # Type: Optional[aiomysql.Connection] self._transaction_class = type( 'TransactionWrapper', (TransactionWrapper, self.__class__), {} ) - async def create_connection(self): + async def create_connection(self) -> None: try: if not self.single_connection: self._db_pool = await aiomysql.create_pool(db=self.database, **self.template) @@ -59,13 +77,13 @@ async def create_connection(self): ) ) - async def close(self): - if not self.single_connection: + async def close(self) -> None: + if self._db_pool: self._db_pool.close() - else: + if self._connection: self._connection.close() - async def db_create(self): + async def db_create(self) -> None: single_connection = self.single_connection self.single_connection = True self._connection = await aiomysql.connect( @@ -74,10 +92,10 @@ async def db_create(self): await self.execute_script( 'CREATE DATABASE {}'.format(self.database) ) - self._connection.close() + self._connection.close() # type: ignore self.single_connection = single_connection - async def db_delete(self): + async def db_delete(self) -> None: single_connection = self.single_connection self.single_connection = True self._connection = await aiomysql.connect( @@ -87,7 +105,7 @@ async def db_delete(self): await self.execute_script('DROP DATABASE {}'.format(self.database)) except pymysql.err.DatabaseError: pass - self._connection.close() + self._connection.close() # type: ignore self.single_connection = single_connection def acquire_connection(self): @@ -102,36 +120,25 @@ def _in_transaction(self): else: return self._transaction_class(self.connection_name, pool=self._db_pool) - async def execute_query(self, query, get_inserted_id=False): - try: - async with self.acquire_connection() as connection: - async with connection.cursor(aiomysql.DictCursor) as cursor: - self.log.debug(query) - - await cursor.execute(query) - if get_inserted_id: - return cursor.lastrowid # return auto-generated id - result = await cursor.fetchall() - return result - except (pymysql.err.OperationalError, pymysql.err.ProgrammingError, - pymysql.err.DataError, pymysql.err.InternalError, - pymysql.err.NotSupportedError) as exc: - raise OperationalError(exc) - except pymysql.err.IntegrityError as exc: - raise IntegrityError(exc) - - async def execute_script(self, script): - try: - async with self.acquire_connection() as connection: - async with connection.cursor() as cursor: - self.log.debug(script) - await cursor.execute(script) - except (pymysql.err.OperationalError, pymysql.err.ProgrammingError, - pymysql.err.DataError, pymysql.err.InternalError, - pymysql.err.NotSupportedError) as exc: - raise OperationalError(exc) - except pymysql.err.IntegrityError as exc: - raise IntegrityError(exc) + @translate_exceptions + async def execute_insert(self, query: str) -> int: + async with self.acquire_connection() as connection: + async with connection.cursor() as cursor: + await cursor.execute(query) + return cursor.lastrowid # return auto-generated id + + @translate_exceptions + async def execute_query(self, query: str) -> List[aiomysql.DictCursor]: + async with self.acquire_connection() as connection: + async with connection.cursor(aiomysql.DictCursor) as cursor: + await cursor.execute(query) + return await cursor.fetchall() + + @translate_exceptions + async def execute_script(self, query: str) -> None: + async with self.acquire_connection() as connection: + async with connection.cursor() as cursor: + await cursor.execute(query) async def get_single_connection(self): if self.single_connection: diff --git a/tortoise/backends/mysql/executor.py b/tortoise/backends/mysql/executor.py index 37eb389e8..ad410b7ce 100644 --- a/tortoise/backends/mysql/executor.py +++ b/tortoise/backends/mysql/executor.py @@ -60,7 +60,7 @@ async def execute_insert(self, instance): .insert(*values) ) - instance.id = await self.connection.execute_query(query, get_inserted_id=True) + instance.id = await self.connection.execute_insert(query) await self.db.release_single_connection(self.connection) self.connection = None return instance diff --git a/tortoise/backends/sqlite/client.py b/tortoise/backends/sqlite/client.py index c4c0c830c..ba013efc5 100644 --- a/tortoise/backends/sqlite/client.py +++ b/tortoise/backends/sqlite/client.py @@ -1,6 +1,8 @@ import logging import os import sqlite3 +from functools import wraps +from typing import List, Optional # noqa import aiosqlite @@ -12,89 +14,94 @@ from tortoise.transactions import current_transaction_map +def translate_exceptions(func): + @wraps(func) + async def wrapped(self, query): + self.log.debug(query) + try: + return await func(self, query) + except sqlite3.OperationalError as exc: + raise OperationalError(exc) + except sqlite3.IntegrityError as exc: + raise IntegrityError(exc) + return wrapped + + class SqliteClient(BaseDBAsyncClient): executor_class = SqliteExecutor schema_generator = SqliteSchemaGenerator - def __init__(self, file_path, **kwargs): + def __init__(self, file_path: str, **kwargs) -> None: super().__init__(**kwargs) self.filename = file_path self._transaction_class = type( 'TransactionWrapper', (TransactionWrapper, self.__class__), {} ) - self._connection = None + self._connection = None # type: Optional[aiosqlite.Connection] - async def create_connection(self): + async def create_connection(self) -> None: if not self._connection: # pragma: no branch self._connection = aiosqlite.connect(self.filename, isolation_level=None) self._connection.start() await self._connection._connect() + self._connection._conn.row_factory = sqlite3.Row - async def close(self): + async def close(self) -> None: if self._connection: await self._connection.close() self._connection = None - async def db_create(self): + async def db_create(self) -> None: pass - async def db_delete(self): + async def db_delete(self) -> None: await self.close() try: os.remove(self.filename) except FileNotFoundError: # pragma: nocoverage pass - def acquire_connection(self): + def acquire_connection(self) -> ConnectionWrapper: return ConnectionWrapper(self._connection) - def _in_transaction(self): + def _in_transaction(self) -> 'TransactionWrapper': return self._transaction_class(self.connection_name, connection=self._connection) - async def execute_query(self, query, get_inserted_id=False): - self.log.debug(query) - try: - async with self.acquire_connection() as connection: - connection._conn.row_factory = sqlite3.Row - cursor = await connection.execute(query) - results = await cursor.fetchall() - if get_inserted_id: - await cursor.execute('SELECT last_insert_rowid()') - inserted_id = await cursor.fetchone() - return inserted_id - return [dict(row) for row in results] - except sqlite3.OperationalError as exc: - raise OperationalError(exc) - except sqlite3.IntegrityError as exc: - raise IntegrityError(exc) - - async def execute_script(self, script): - connection = self._connection - self.log.debug(script) - try: - await connection.executescript(script) - except sqlite3.OperationalError as exc: - raise OperationalError(exc) - except sqlite3.IntegrityError as exc: - raise IntegrityError(exc) - - async def get_single_connection(self): + @translate_exceptions + async def execute_insert(self, query: str) -> int: + async with self.acquire_connection() as connection: + cursor = await connection.execute(query) + await cursor.execute('SELECT last_insert_rowid()') + return (await cursor.fetchone())[0] + + @translate_exceptions + async def execute_query(self, query: str) -> List[dict]: + async with self.acquire_connection() as connection: + cursor = await connection.execute(query) + return [dict(row) for row in await cursor.fetchall()] + + @translate_exceptions + async def execute_script(self, query: str) -> None: + async with self.acquire_connection() as connection: + await connection.executescript(query) + + async def get_single_connection(self) -> 'SqliteClient': return self - async def release_single_connection(self, single_connection): + async def release_single_connection(self, single_connection: BaseDBAsyncClient) -> None: pass class TransactionWrapper(SqliteClient, BaseTransactionWrapper): - def __init__(self, connection_name, connection): + def __init__(self, connection_name: str, connection: aiosqlite.Connection) -> None: self.connection_name = connection_name - self._connection = connection + self._connection = connection # type: aiosqlite.Connection self.log = logging.getLogger('db_client') self._transaction_class = self.__class__ self._old_context_value = None self._finalized = False - async def start(self): + async def start(self) -> None: try: await self._connection.commit() await self._connection.execute('BEGIN') @@ -104,14 +111,14 @@ async def start(self): self._old_context_value = current_transaction.get() current_transaction.set(self) - async def rollback(self): + async def rollback(self) -> None: if self._finalized: raise TransactionManagementError('Transaction already finalised') self._finalized = True await self._connection.rollback() current_transaction_map[self.connection_name].set(self._old_context_value) - async def commit(self): + async def commit(self) -> None: if self._finalized: raise TransactionManagementError('Transaction already finalised') self._finalized = True diff --git a/tortoise/backends/sqlite/executor.py b/tortoise/backends/sqlite/executor.py index 81d4e8b9b..d37450310 100644 --- a/tortoise/backends/sqlite/executor.py +++ b/tortoise/backends/sqlite/executor.py @@ -40,8 +40,7 @@ async def execute_insert(self, instance): .insert(*values) ) - result = await self.connection.execute_query(query, get_inserted_id=True) - instance.id = result[0] + instance.id = await self.connection.execute_insert(query) await self.db.release_single_connection(self.connection) self.connection = None return instance From 02d2096948d668d7b374acbaf7c6e19d4fac9897 Mon Sep 17 00:00:00 2001 From: Grigi Date: Thu, 4 Oct 2018 11:50:43 +0200 Subject: [PATCH 3/9] Use prepared statements for insert, refactor insert to be more generic --- tortoise/backends/asyncpg/client.py | 14 ++++++--- tortoise/backends/asyncpg/executor.py | 21 ++++--------- tortoise/backends/base/client.py | 4 +-- tortoise/backends/base/executor.py | 27 ++++++++++++---- tortoise/backends/mysql/client.py | 13 +++++--- tortoise/backends/mysql/executor.py | 45 +++++++++------------------ tortoise/backends/sqlite/client.py | 12 ++++--- tortoise/backends/sqlite/executor.py | 20 +++--------- 8 files changed, 73 insertions(+), 83 deletions(-) diff --git a/tortoise/backends/asyncpg/client.py b/tortoise/backends/asyncpg/client.py index 46ab68d16..9efc20566 100644 --- a/tortoise/backends/asyncpg/client.py +++ b/tortoise/backends/asyncpg/client.py @@ -16,10 +16,9 @@ def translate_exceptions(func): @wraps(func) - async def wrapped(self, query): - self.log.debug(query) + async def wrapped(self, query, *args): try: - return await func(self, query) + return await func(self, query, *args) except asyncpg.exceptions.SyntaxOrAccessError as exc: raise OperationalError(exc) except asyncpg.exceptions.IntegrityConstraintViolationError as exc: @@ -124,17 +123,22 @@ def _in_transaction(self) -> 'TransactionWrapper': return self._transaction_class(self.connection_name, pool=self._db_pool) @translate_exceptions - async def execute_insert(self, query: str) -> int: + async def execute_insert(self, query: str, values: list) -> int: + self.log.debug('%s: %s', query, values) async with self.acquire_connection() as connection: - return (await connection.fetch(query))[0][0] + # TODO: Cache prepared statement + stmt = await connection.prepare(query) + return (await stmt.fetchval(*values)) @translate_exceptions async def execute_query(self, query: str) -> List[dict]: + self.log.debug(query) async with self.acquire_connection() as connection: return await connection.fetch(query) @translate_exceptions async def execute_script(self, query: str) -> None: + self.log.debug(query) async with self.acquire_connection() as connection: await connection.execute(query) diff --git a/tortoise/backends/asyncpg/executor.py b/tortoise/backends/asyncpg/executor.py index 8c8517b39..91becc7db 100644 --- a/tortoise/backends/asyncpg/executor.py +++ b/tortoise/backends/asyncpg/executor.py @@ -1,22 +1,13 @@ +from typing import List + from pypika import Table from tortoise.backends.base.executor import BaseExecutor class AsyncpgExecutor(BaseExecutor): - async def execute_insert(self, instance): - self.connection = await self.db.get_single_connection() - regular_columns, columns = self._prepare_insert_columns() - values = self._prepare_insert_values( - instance=instance, - regular_columns=regular_columns, - ) - - query = str( + def _prepare_insert_statement(self, columns: List[str]) -> str: + return str( self.connection.query_class.into(Table(self.model._meta.table)).columns(*columns) - .insert(*values).returning('id') - ) - instance.id = await self.connection.execute_insert(query) - await self.db.release_single_connection(self.connection) - self.connection = None - return instance + .insert('???').returning('id') + ).replace("'???'", ','.join(['$%d' % (i + 1, ) for i in range(len(columns))])) diff --git a/tortoise/backends/base/client.py b/tortoise/backends/base/client.py index 3e6df629e..6c429b913 100644 --- a/tortoise/backends/base/client.py +++ b/tortoise/backends/base/client.py @@ -12,7 +12,7 @@ class BaseDBAsyncClient: executor_class = BaseExecutor schema_generator = BaseSchemaGenerator - def __init__(self, connection_name: str, single_connection: bool=True, **kwargs) -> None: + def __init__(self, connection_name: str, single_connection: bool = True, **kwargs) -> None: self.log = logging.getLogger('db_client') self.single_connection = single_connection self.connection_name = connection_name @@ -38,7 +38,7 @@ def acquire_connection(self) -> 'ConnectionWrapper': def _in_transaction(self) -> 'BaseTransactionWrapper': raise NotImplementedError() # pragma: nocoverage - async def execute_insert(self, query: str) -> int: + async def execute_insert(self, query: str, values: list) -> int: raise NotImplementedError() # pragma: nocoverage async def execute_query(self, query: str) -> Sequence[dict]: diff --git a/tortoise/backends/base/executor.py b/tortoise/backends/base/executor.py index 46bd32682..158dd1123 100644 --- a/tortoise/backends/base/executor.py +++ b/tortoise/backends/base/executor.py @@ -1,4 +1,4 @@ -from typing import Callable, Dict, Type # noqa +from typing import Callable, Dict, List, Type # noqa from pypika import Table @@ -8,6 +8,7 @@ class BaseExecutor: TO_DB_OVERRIDE = {} # type: Dict[Type[fields.Field], Callable] + FILTER_FUNC_OVERRIDE = {} # type: Dict[Callable, Callable] def __init__(self, model, db=None, prefetch_map=None, prefetch_queries=None): self.model = model @@ -49,12 +50,27 @@ def _prepare_insert_values(self, instance, regular_columns): return [self._field_to_db(self.model._meta.fields_map[column], getattr(instance, column), instance) for column in regular_columns] - async def execute_insert(self, instance): + def _prepare_insert_statement(self, columns: List[str]) -> str: # Insert should implement returning new id to saved object # Each db has it's own methods for it, so each implementation should # go to descendant executors raise NotImplementedError() # pragma: nocoverage + async def execute_insert(self, instance): + self.connection = await self.db.get_single_connection() + + regular_columns, columns = self._prepare_insert_columns() + query = self._prepare_insert_statement(columns) + + values = self._prepare_insert_values( + instance=instance, + regular_columns=regular_columns, + ) + instance.id = await self.connection.execute_insert(query, values) + await self.db.release_single_connection(self.connection) + self.connection = None + return instance + async def execute_update(self, instance): self.connection = await self.db.get_single_connection() table = Table(self.model._meta.table) @@ -204,7 +220,6 @@ async def fetch_for_list(self, instance_list, *args): self.connection = None return instance_list - @staticmethod - def get_overridden_filter_func(filter_func): - # Returns None if no filters was overridden - return + @classmethod + def get_overridden_filter_func(cls, filter_func): + return cls.FILTER_FUNC_OVERRIDE.get(filter_func) diff --git a/tortoise/backends/mysql/client.py b/tortoise/backends/mysql/client.py index 2d0cc607d..37af43cba 100644 --- a/tortoise/backends/mysql/client.py +++ b/tortoise/backends/mysql/client.py @@ -17,10 +17,9 @@ def translate_exceptions(func): @wraps(func) - async def wrapped(self, query): - self.log.debug(query) + async def wrapped(self, query, *args): try: - return await func(self, query) + return await func(self, query, *args) except (pymysql.err.OperationalError, pymysql.err.ProgrammingError, pymysql.err.DataError, pymysql.err.InternalError, pymysql.err.NotSupportedError) as exc: @@ -121,14 +120,17 @@ def _in_transaction(self): return self._transaction_class(self.connection_name, pool=self._db_pool) @translate_exceptions - async def execute_insert(self, query: str) -> int: + async def execute_insert(self, query: str, values: list) -> int: + self.log.debug('%s: %s', query, values) async with self.acquire_connection() as connection: async with connection.cursor() as cursor: - await cursor.execute(query) + # TODO: Use prepared statement, and cache it + await cursor.execute(query, values) return cursor.lastrowid # return auto-generated id @translate_exceptions async def execute_query(self, query: str) -> List[aiomysql.DictCursor]: + self.log.debug(query) async with self.acquire_connection() as connection: async with connection.cursor(aiomysql.DictCursor) as cursor: await cursor.execute(query) @@ -136,6 +138,7 @@ async def execute_query(self, query: str) -> List[aiomysql.DictCursor]: @translate_exceptions async def execute_script(self, query: str) -> None: + self.log.debug(query) async with self.acquire_connection() as connection: async with connection.cursor() as cursor: await cursor.execute(query) diff --git a/tortoise/backends/mysql/executor.py b/tortoise/backends/mysql/executor.py index ad410b7ce..bf0ed7632 100644 --- a/tortoise/backends/mysql/executor.py +++ b/tortoise/backends/mysql/executor.py @@ -1,3 +1,5 @@ +from typing import List + from pypika import MySQLQuery, Table, functions from pypika.enums import SqlTypes @@ -36,35 +38,18 @@ def mysql_insensitive_ends_with(field, value): ) -FILTER_FUNC_OVERRIDE = { - contains: mysql_contains, - starts_with: mysql_starts_with, - ends_with: mysql_ends_with, - insensitive_contains: mysql_insensitive_contains, - insensitive_starts_with: mysql_insensitive_starts_with, - insensitive_ends_with: mysql_insensitive_ends_with -} - - class MySQLExecutor(BaseExecutor): - async def execute_insert(self, instance): - self.connection = await self.db.get_single_connection() - regular_columns, columns = self._prepare_insert_columns() - values = self._prepare_insert_values( - instance=instance, - regular_columns=regular_columns, - ) - - query = str( + FILTER_FUNC_OVERRIDE = { + contains: mysql_contains, + starts_with: mysql_starts_with, + ends_with: mysql_ends_with, + insensitive_contains: mysql_insensitive_contains, + insensitive_starts_with: mysql_insensitive_starts_with, + insensitive_ends_with: mysql_insensitive_ends_with + } + + def _prepare_insert_statement(self, columns: List[str]) -> str: + return str( MySQLQuery.into(Table(self.model._meta.table)).columns(*columns) - .insert(*values) - ) - - instance.id = await self.connection.execute_insert(query) - await self.db.release_single_connection(self.connection) - self.connection = None - return instance - - @staticmethod - def get_overridden_filter_func(filter_func): - return FILTER_FUNC_OVERRIDE.get(filter_func) + .insert('???') + ).replace("'???'", ','.join(['%s' for _ in range(len(columns))])) diff --git a/tortoise/backends/sqlite/client.py b/tortoise/backends/sqlite/client.py index ba013efc5..c967b9d58 100644 --- a/tortoise/backends/sqlite/client.py +++ b/tortoise/backends/sqlite/client.py @@ -16,10 +16,9 @@ def translate_exceptions(func): @wraps(func) - async def wrapped(self, query): - self.log.debug(query) + async def wrapped(self, query, *args): try: - return await func(self, query) + return await func(self, query, *args) except sqlite3.OperationalError as exc: raise OperationalError(exc) except sqlite3.IntegrityError as exc: @@ -68,20 +67,23 @@ def _in_transaction(self) -> 'TransactionWrapper': return self._transaction_class(self.connection_name, connection=self._connection) @translate_exceptions - async def execute_insert(self, query: str) -> int: + async def execute_insert(self, query: str, values: list) -> int: + self.log.debug('%s: %s', query, values) async with self.acquire_connection() as connection: - cursor = await connection.execute(query) + cursor = await connection.execute(query, values) await cursor.execute('SELECT last_insert_rowid()') return (await cursor.fetchone())[0] @translate_exceptions async def execute_query(self, query: str) -> List[dict]: + self.log.debug(query) async with self.acquire_connection() as connection: cursor = await connection.execute(query) return [dict(row) for row in await cursor.fetchall()] @translate_exceptions async def execute_script(self, query: str) -> None: + self.log.debug(query) async with self.acquire_connection() as connection: await connection.executescript(query) diff --git a/tortoise/backends/sqlite/executor.py b/tortoise/backends/sqlite/executor.py index d37450310..4e0079ece 100644 --- a/tortoise/backends/sqlite/executor.py +++ b/tortoise/backends/sqlite/executor.py @@ -1,4 +1,5 @@ from decimal import Decimal +from typing import List from pypika import Table @@ -28,19 +29,8 @@ class SqliteExecutor(BaseExecutor): fields.DecimalField: to_db_decimal, } - async def execute_insert(self, instance): - self.connection = await self.db.get_single_connection() - regular_columns, columns = self._prepare_insert_columns() - values = self._prepare_insert_values( - instance=instance, - regular_columns=regular_columns, - ) - query = str( + def _prepare_insert_statement(self, columns: List[str]) -> str: + return str( self.connection.query_class.into(Table(self.model._meta.table)).columns(*columns) - .insert(*values) - ) - - instance.id = await self.connection.execute_insert(query) - await self.db.release_single_connection(self.connection) - self.connection = None - return instance + .insert('???') + ).replace("'???'", ','.join(['?' for _ in range(len(columns))])) From 4ed63c211ecc059a65e5a010dcf7fc9c5d38d281 Mon Sep 17 00:00:00 2001 From: Grigi Date: Thu, 4 Oct 2018 12:15:43 +0200 Subject: [PATCH 4/9] Simple insert cache --- tortoise/backends/base/executor.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tortoise/backends/base/executor.py b/tortoise/backends/base/executor.py index 158dd1123..c643ceb80 100644 --- a/tortoise/backends/base/executor.py +++ b/tortoise/backends/base/executor.py @@ -1,10 +1,12 @@ -from typing import Callable, Dict, List, Type # noqa +from typing import Callable, Dict, List, Tuple, Type # noqa from pypika import Table from tortoise import fields from tortoise.exceptions import OperationalError +INSERT_CACHE = {} # type: Dict[str, Tuple[list, list, str]] + class BaseExecutor: TO_DB_OVERRIDE = {} # type: Dict[Type[fields.Field], Callable] @@ -58,9 +60,13 @@ def _prepare_insert_statement(self, columns: List[str]) -> str: async def execute_insert(self, instance): self.connection = await self.db.get_single_connection() - - regular_columns, columns = self._prepare_insert_columns() - query = self._prepare_insert_statement(columns) + key = '%s:%s' % (self.db.connection_name, self.model._meta.table) + if key not in INSERT_CACHE: + regular_columns, columns = self._prepare_insert_columns() + query = self._prepare_insert_statement(columns) + INSERT_CACHE[key] = regular_columns, columns, query + else: + regular_columns, columns, query = INSERT_CACHE[key] values = self._prepare_insert_values( instance=instance, From fbd6de0154b8602b0b879bcf325e7ef7e96a34a8 Mon Sep 17 00:00:00 2001 From: Grigi Date: Thu, 4 Oct 2018 12:26:41 +0200 Subject: [PATCH 5/9] lint --- docs/conf.py | 2 +- tortoise/backends/asyncpg/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 28a2f3df2..587ff2843 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -33,7 +33,7 @@ def __getattr__(cls, name): project = 'Tortoise' -copyright = '2018, Andrey Bondar' +copyright = '2018, Andrey Bondar' # pylint: disable=W0622 author = 'Andrey Bondar' diff --git a/tortoise/backends/asyncpg/client.py b/tortoise/backends/asyncpg/client.py index 9efc20566..44db99916 100644 --- a/tortoise/backends/asyncpg/client.py +++ b/tortoise/backends/asyncpg/client.py @@ -128,7 +128,7 @@ async def execute_insert(self, query: str, values: list) -> int: async with self.acquire_connection() as connection: # TODO: Cache prepared statement stmt = await connection.prepare(query) - return (await stmt.fetchval(*values)) + return await stmt.fetchval(*values) @translate_exceptions async def execute_query(self, query: str) -> List[dict]: From dbf7646a1cfac0dacb2fd280126c7d4a4f8512cb Mon Sep 17 00:00:00 2001 From: Grigi Date: Sat, 13 Oct 2018 14:14:15 +0200 Subject: [PATCH 6/9] use aiosqlite>=0.7 --- .isort.cfg | 2 +- requirements-dev.txt | 31 +++++++++++++++--------------- requirements.txt | 2 +- tortoise/backends/sqlite/client.py | 7 ++----- tortoise/fields.py | 2 +- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 7cd5b89c4..da74cdb60 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,5 +1,5 @@ [settings] line_length=100 multi_line_output=0 -known_third_party= +known_third_party=aiosqlite,ciso8601 not_skip=__init__.py diff --git a/requirements-dev.txt b/requirements-dev.txt index dbe0583c6..4d54b369e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,8 +7,8 @@ aenum==2.0.8 # via pypika aiocontextvars==0.1.2 ; python_version < "3.7" aiomysql==0.0.19 -aiosqlite==0.6.0 -alabaster==0.7.11 # via sphinx +aiosqlite==0.7.0 +alabaster==0.7.12 # via sphinx asn1crypto==0.24.0 # via cryptography astroid==2.0.4 # via pylint asyncpg==0.17.0 @@ -18,20 +18,21 @@ bandit==1.5.1 certifi==2018.8.24 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests -ciso8601==2.0.1 +ciso8601==2.1.1 click==7.0 # via pip-tools cloud-sptheme==1.9.4 -colorama==0.3.9 # via green +colorama==0.4.0 # via green coverage==4.5.1 # via coveralls, green -coveralls==1.5.0 +coveralls==1.5.1 cryptography==2.3.1 # via pymysql docopt==0.6.2 # via coveralls docutils==0.14 +filelock==3.0.9 # via tox flake8-isort==2.5 flake8==3.5.0 # via flake8-isort -gitdb2==2.0.4 # via gitpython +gitdb2==2.0.5 # via gitpython gitpython==2.1.11 # via bandit -green==2.12.1 +green==2.13.0 idna==2.7 # via cryptography, requests imagesize==1.1.0 # via sphinx isort==4.3.4 # via flake8-isort, pylint @@ -42,23 +43,23 @@ mccabe==0.6.1 # via flake8, pylint mypy-extensions==0.4.1 # via mypy mypy==0.630 packaging==18.0 # via sphinx -pbr==4.2.0 # via stevedore -pip-tools==3.0.0 +pbr==4.3.0 # via stevedore +pip-tools==3.1.0 pluggy==0.7.1 # via tox -py==1.6.0 # via tox +py==1.7.0 # via tox pycodestyle==2.3.1 # via flake8 pycparser==2.19 # via cffi pyflakes==1.6.0 # via flake8 pygments==2.2.0 pylint==2.1.1 pymysql==0.9.2 # via aiomysql -pyparsing==2.2.1 # via packaging -pypika==0.15.6 +pyparsing==2.2.2 # via packaging +pypika==0.15.7 pytz==2018.5 # via babel pyyaml==3.13 # via bandit requests==2.19.1 # via coveralls, sphinx six==1.11.0 # via astroid, bandit, cryptography, packaging, pip-tools, sphinx, stevedore, tox -smmap2==2.0.4 # via gitdb2 +smmap2==2.0.5 # via gitdb2 snowballstemmer==1.2.1 # via sphinx sphinx-autodoc-typehints==1.3.0 sphinx==1.8.1 @@ -66,8 +67,8 @@ sphinxcontrib-websupport==1.1.0 # via sphinx stevedore==1.29.0 # via bandit termstyle==0.1.11 # via green testfixtures==6.3.0 # via flake8-isort -toml==0.9.6 # via tox -tox==3.4.0 +toml==0.10.0 # via tox +tox==3.5.2 typed-ast==1.1.0 # via astroid, mypy unidecode==1.0.22 # via green urllib3==1.23 # via requests diff --git a/requirements.txt b/requirements.txt index fe4ad85eb..3a87367cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ pypika>=0.15.6,<1.0 ciso8601>=2.0 aiocontextvars==0.1.2;python_version<"3.7" -aiosqlite>=0.6.0 +aiosqlite>=0.7.0 diff --git a/tortoise/backends/sqlite/client.py b/tortoise/backends/sqlite/client.py index c967b9d58..caa2ff25c 100644 --- a/tortoise/backends/sqlite/client.py +++ b/tortoise/backends/sqlite/client.py @@ -70,16 +70,13 @@ def _in_transaction(self) -> 'TransactionWrapper': async def execute_insert(self, query: str, values: list) -> int: self.log.debug('%s: %s', query, values) async with self.acquire_connection() as connection: - cursor = await connection.execute(query, values) - await cursor.execute('SELECT last_insert_rowid()') - return (await cursor.fetchone())[0] + return (await connection.execute_insert(query, values))[0] @translate_exceptions async def execute_query(self, query: str) -> List[dict]: self.log.debug(query) async with self.acquire_connection() as connection: - cursor = await connection.execute(query) - return [dict(row) for row in await cursor.fetchall()] + return [dict(row) for row in await connection.execute_fetchall(query)] @translate_exceptions async def execute_script(self, query: str) -> None: diff --git a/tortoise/fields.py b/tortoise/fields.py index 55067bb2a..324978ebf 100644 --- a/tortoise/fields.py +++ b/tortoise/fields.py @@ -4,9 +4,9 @@ from decimal import Decimal from typing import Any, Optional +import ciso8601 from pypika import Table -import ciso8601 from tortoise.exceptions import ConfigurationError, NoValuesFetched, OperationalError from tortoise.utils import QueryAsyncIterator From c39b21c2ea993ed3c0f746fdc2b8a9e9b9135cf1 Mon Sep 17 00:00:00 2001 From: Grigi Date: Sun, 14 Oct 2018 17:01:56 +0200 Subject: [PATCH 7/9] pre-generate initial pypika query object per model, as it is immutable --- tortoise/__init__.py | 8 ++++++++ tortoise/models.py | 4 +++- tortoise/queryset.py | 6 +----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tortoise/__init__.py b/tortoise/__init__.py index 32a334724..5ecf64088 100644 --- a/tortoise/__init__.py +++ b/tortoise/__init__.py @@ -206,6 +206,12 @@ def _get_config_from_config_file(cls, config_file): ) return config + @classmethod + def _build_initial_querysets(cls): + for app in cls.apps.values(): + for model in app.values(): + model._meta.basequery = model._meta.db.query_class.from_(model._meta.table) + @classmethod async def init( cls, @@ -303,6 +309,8 @@ async def init( cls._init_relations() + cls._build_initial_querysets() + cls._inited = True @classmethod diff --git a/tortoise/models.py b/tortoise/models.py index d5337d89d..b909ccf2d 100644 --- a/tortoise/models.py +++ b/tortoise/models.py @@ -218,7 +218,8 @@ def get_filters_for_field(field_name: str, field: Optional[fields.Field], source class MetaInfo: __slots__ = ('abstract', 'table', 'app', 'fields', 'db_fields', 'm2m_fields', 'fk_fields', 'backward_fk_fields', 'fetch_fields', 'fields_db_projection', '_inited', - 'fields_db_projection_reverse', 'filters', 'fields_map', 'default_connection') + 'fields_db_projection_reverse', 'filters', 'fields_map', 'default_connection', + 'basequery') def __init__(self, meta): self.abstract = getattr(meta, 'abstract', False) # type: bool @@ -236,6 +237,7 @@ def __init__(self, meta): self.fields_map = {} # type: Dict[str, fields.Field] self._inited = False self.default_connection = None + self.basequery = None @property def db(self): diff --git a/tortoise/queryset.py b/tortoise/queryset.py index b57480a29..217f25450 100644 --- a/tortoise/queryset.py +++ b/tortoise/queryset.py @@ -142,11 +142,7 @@ def __init__(self, model) -> None: self.fields = model._meta.db_fields self.model = model - if not hasattr(model._meta.db, 'query_class'): - # do not build Query if Tortoise wasn't inited - self.query = None - else: - self.query = model._meta.db.query_class.from_(model._meta.table) + self.query = model._meta.basequery self._prefetch_map = {} # type: Dict[str, Set[str]] self._prefetch_queries = {} # type: Dict[str, QuerySet] From 57dac1e3edd631e9f68f9b5e9eda47c5a1dd00ac Mon Sep 17 00:00:00 2001 From: Grigi Date: Mon, 15 Oct 2018 10:47:39 +0200 Subject: [PATCH 8/9] v0.10.9 --- CHANGELOG.rst | 6 ++++++ tortoise/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c6f66046b..6daf46efb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Changelog ========= +0.10.9 +------ +- Uses macros on SQLite driver to minimise syncronisation. ``aiosqlite>=0.7.0`` +- Uses prepared statements for insert, large insert performance increase. +- Pre-generate base pypika query object per model, providing general purpose speedup. + 0.10.8 ------ - Performance fixes from ``pypika>=0.15.6`` diff --git a/tortoise/__init__.py b/tortoise/__init__.py index 5ecf64088..6dc7df40f 100644 --- a/tortoise/__init__.py +++ b/tortoise/__init__.py @@ -382,4 +382,4 @@ async def do_stuff(): loop.run_until_complete(Tortoise.close_connections()) -__version__ = "0.10.8" +__version__ = "0.10.9" From ebfd16d03aed89801f00ea2e64b1eb26755dee95 Mon Sep 17 00:00:00 2001 From: Grigi Date: Tue, 16 Oct 2018 19:37:37 +0200 Subject: [PATCH 9/9] use .format() instead --- tortoise/backends/base/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tortoise/backends/base/executor.py b/tortoise/backends/base/executor.py index c643ceb80..a29a8769f 100644 --- a/tortoise/backends/base/executor.py +++ b/tortoise/backends/base/executor.py @@ -60,7 +60,7 @@ def _prepare_insert_statement(self, columns: List[str]) -> str: async def execute_insert(self, instance): self.connection = await self.db.get_single_connection() - key = '%s:%s' % (self.db.connection_name, self.model._meta.table) + key = '{}:{}'.format(self.db.connection_name, self.model._meta.table) if key not in INSERT_CACHE: regular_columns, columns = self._prepare_insert_columns() query = self._prepare_insert_statement(columns)