Skip to content

Commit

Permalink
[WIP] Generate schema correctly with non-default column/table names. (#…
Browse files Browse the repository at this point in the history
…166)

* Fixed generation of schema for fields that defined custom ``source_field`` values defined
* Fixed working with Models that have fields with custom ``source_field`` values defined
* Tests for generate schema
* Tests for basic functionality: get/filter/fk/rfk/m2m
  • Loading branch information
grigi authored Aug 14, 2019
1 parent 9399419 commit 006be72
Show file tree
Hide file tree
Showing 14 changed files with 377 additions and 63 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ Bugfixes:
- Fixed missing table/column comment generation for ``ForeignKeyField`` and ``ManyToManyField``
- Fixed comment generation to escape properly for ``SQLite``
- Fixed comment generation for ``PostgreSQL`` to not duplicate comments
- Fixed generation of schema for fields that defined custom ``source_field`` values defined
- Fixed working with Models that have fields with custom ``source_field`` values defined

Docs/examples:
^^^^^^^^^^^^^^
Expand Down
9 changes: 7 additions & 2 deletions tortoise/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

logger = logging.getLogger("tortoise")

if sys.version_info < (3, 6):
if sys.version_info < (3, 6): # pragma: nocoverage
warnings.warn("Tortoise-ORM is soon going to require Python 3.6", DeprecationWarning)


Expand Down Expand Up @@ -99,14 +99,19 @@ def split_reference(reference: str) -> Tuple[str, str]:
related_model = get_related_model(related_app_name, related_model_name)

key_field = "{}_id".format(field)
fk_object.source_field = key_field
key_fk_object = deepcopy(related_model._meta.pk)
key_fk_object.pk = False
key_fk_object.index = fk_object.index
key_fk_object.default = fk_object.default
key_fk_object.null = fk_object.null
key_fk_object.generated = fk_object.generated
key_fk_object.reference = fk_object
if fk_object.source_field:
key_fk_object.source_field = fk_object.source_field
fk_object.source_field = key_field
else:
fk_object.source_field = key_field
key_fk_object.source_field = key_field
model._meta.add_field(key_field, key_fk_object)

fk_object.type = related_model
Expand Down
3 changes: 2 additions & 1 deletion tortoise/backends/asyncpg/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,6 @@ def _prepare_insert_statement(self, columns: List[str]) -> str:
async def _process_insert_result(self, instance: Model, results: Optional[asyncpg.Record]):
if results:
generated_fields = self.model._meta.generated_db_fields
db_projection = instance._meta.fields_db_projection_reverse
for key, val in zip(generated_fields, results):
setattr(instance, key, val)
setattr(instance, db_projection[key], val)
6 changes: 4 additions & 2 deletions tortoise/backends/asyncpg/schema_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ def _column_comment_generator(self, table: str, column: str, comment: str) -> st
return ""

def _post_table_hook(self) -> str:
val = "\n" + "\n".join(self.comments_array)
val = "\n".join(self.comments_array)
self.comments_array = []
return val
if val:
return "\n" + val
return ""
2 changes: 1 addition & 1 deletion tortoise/backends/base/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ async def _prefetch_m2m_relation(self, instance_list: list, field: str, related_
)
for e in raw_results
}
related_object_list = [related_query.model(_from_db=True, **e) for e in raw_results]
related_object_list = [related_query.model._init_from_db(**e) for e in raw_results]
await self.__class__(
model=related_query.model, db=self.db, prefetch_map=related_query._prefetch_map
).fetch_for_list(related_object_list)
Expand Down
19 changes: 11 additions & 8 deletions tortoise/backends/base/schema_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,13 @@ class BaseSchemaGenerator:
FIELD_TEMPLATE = '"{name}" {type} {nullable} {unique}{primary}{comment}'
INDEX_CREATE_TEMPLATE = 'CREATE INDEX {exists}"{index_name}" ON "{table_name}" ({fields});'
UNIQUE_CONSTRAINT_CREATE_TEMPLATE = "UNIQUE ({fields})"
FK_TEMPLATE = ' REFERENCES "{table}" (id) ON DELETE {on_delete}{comment}'
FK_TEMPLATE = ' REFERENCES "{table}" ({field}) ON DELETE {on_delete}{comment}'
M2M_TABLE_TEMPLATE = (
'CREATE TABLE {exists}"{table_name}" (\n'
' "{backward_key}" {backward_type} NOT NULL REFERENCES "{backward_table}" (id)'
" ON DELETE CASCADE,\n"
' "{forward_key}" {forward_type} NOT NULL REFERENCES "{forward_table}" (id)'
" ON DELETE CASCADE\n"
' "{backward_key}" {backward_type} NOT NULL REFERENCES "{backward_table}"'
" ({backward_field}) ON DELETE CASCADE,\n"
' "{forward_key}" {forward_type} NOT NULL REFERENCES "{forward_table}"'
" ({forward_field}) ON DELETE CASCADE\n"
"){comment};"
)

Expand Down Expand Up @@ -145,7 +145,7 @@ def _get_table_sql(self, model, safe=True) -> dict:
else ""
)
if isinstance(field_object, (fields.IntField, fields.BigIntField)) and field_object.pk:
fields_to_create.append(self._get_primary_key_create_string(field_name, comment))
fields_to_create.append(self._get_primary_key_create_string(db_field, comment))
continue
nullable = "NOT NULL" if not field_object.null else ""
unique = "UNIQUE" if field_object.unique else ""
Expand All @@ -170,14 +170,15 @@ def _get_table_sql(self, model, safe=True) -> dict:
)
field_creation_string += self.FK_TEMPLATE.format(
table=field_object.reference.type._meta.table,
field=field_object.reference.type._meta.db_pk_field,
on_delete=field_object.reference.on_delete,
comment=comment,
)
references.add(field_object.reference.type._meta.table)
fields_to_create.append(field_creation_string)

if field_object.index:
fields_with_index.append(field_name)
fields_with_index.append(db_field)

if model._meta.unique_together is not None:
unique_together_sqls = []
Expand Down Expand Up @@ -244,14 +245,16 @@ def _get_table_sql(self, model, safe=True) -> dict:
table_name=field_object.through,
backward_table=model._meta.table,
forward_table=field_object.type._meta.table,
backward_field=model._meta.db_pk_field,
forward_field=field_object.type._meta.db_pk_field,
backward_key=field_object.backward_key,
backward_type=self._get_field_type(model._meta.pk),
forward_key=field_object.forward_key,
forward_type=self._get_field_type(field_object.type._meta.pk),
comment=self._table_comment_generator(
table=field_object.through, comment=field_object.description
)
if model._meta.table_description
if field_object.description
else "",
)
m2m_create_string += self._post_table_hook()
Expand Down
10 changes: 5 additions & 5 deletions tortoise/backends/mysql/schema_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ class MySQLSchemaGenerator(BaseSchemaGenerator):
TABLE_CREATE_TEMPLATE = "CREATE TABLE {exists}`{table_name}` ({fields}){comment};"
INDEX_CREATE_TEMPLATE = "CREATE INDEX `{index_name}` ON `{table_name}` ({fields});"
FIELD_TEMPLATE = "`{name}` {type} {nullable} {unique}{comment}"
FK_TEMPLATE = " REFERENCES `{table}` (`id`) ON DELETE {on_delete}{comment}"
FK_TEMPLATE = " REFERENCES `{table}` (`{field}`) ON DELETE {on_delete}{comment}"
M2M_TABLE_TEMPLATE = (
"CREATE TABLE `{table_name}` (\n"
" `{backward_key}` {backward_type} NOT NULL REFERENCES `{backward_table}` (`id`)"
" ON DELETE CASCADE,\n"
" `{forward_key}` {forward_type} NOT NULL REFERENCES `{forward_table}` (`id`)"
" ON DELETE CASCADE\n"
" `{backward_key}` {backward_type} NOT NULL REFERENCES `{backward_table}`"
" (`{backward_field}`) ON DELETE CASCADE,\n"
" `{forward_key}` {forward_type} NOT NULL REFERENCES `{forward_table}`"
" (`{forward_field}`) ON DELETE CASCADE\n"
"){comment};"
)
FIELD_TYPE_MAP = {
Expand Down
69 changes: 53 additions & 16 deletions tortoise/filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,60 +158,97 @@ def get_filters_for_field(
return get_m2m_filters(field_name, field)
if isinstance(field, fields.BackwardFKRelation):
return get_backward_fk_filters(field_name, field)
actual_field_name = field_name
if field_name == "pk" and field:
actual_field_name = field.model_field_name
return {
field_name: {"field": source_field, "operator": operator.eq},
"{}__not".format(field_name): {"field": source_field, "operator": not_equal},
field_name: {
"field": actual_field_name,
"source_field": source_field,
"operator": operator.eq,
},
"{}__not".format(field_name): {
"field": actual_field_name,
"source_field": source_field,
"operator": not_equal,
},
"{}__in".format(field_name): {
"field": source_field,
"field": actual_field_name,
"source_field": source_field,
"operator": is_in,
"value_encoder": list_encoder,
},
"{}__not_in".format(field_name): {
"field": source_field,
"field": actual_field_name,
"source_field": source_field,
"operator": not_in,
"value_encoder": list_encoder,
},
"{}__isnull".format(field_name): {
"field": source_field,
"field": actual_field_name,
"source_field": source_field,
"operator": is_null,
"value_encoder": bool_encoder,
},
"{}__not_isnull".format(field_name): {
"field": source_field,
"field": actual_field_name,
"source_field": source_field,
"operator": not_null,
"value_encoder": bool_encoder,
},
"{}__gte".format(field_name): {"field": source_field, "operator": operator.ge},
"{}__lte".format(field_name): {"field": source_field, "operator": operator.le},
"{}__gt".format(field_name): {"field": source_field, "operator": operator.gt},
"{}__lt".format(field_name): {"field": source_field, "operator": operator.lt},
"{}__gte".format(field_name): {
"field": actual_field_name,
"source_field": source_field,
"operator": operator.ge,
},
"{}__lte".format(field_name): {
"field": actual_field_name,
"source_field": source_field,
"operator": operator.le,
},
"{}__gt".format(field_name): {
"field": actual_field_name,
"source_field": source_field,
"operator": operator.gt,
},
"{}__lt".format(field_name): {
"field": actual_field_name,
"source_field": source_field,
"operator": operator.lt,
},
"{}__contains".format(field_name): {
"field": source_field,
"field": actual_field_name,
"source_field": source_field,
"operator": contains,
"value_encoder": string_encoder,
},
"{}__startswith".format(field_name): {
"field": source_field,
"field": actual_field_name,
"source_field": source_field,
"operator": starts_with,
"value_encoder": string_encoder,
},
"{}__endswith".format(field_name): {
"field": source_field,
"field": actual_field_name,
"source_field": source_field,
"operator": ends_with,
"value_encoder": string_encoder,
},
"{}__icontains".format(field_name): {
"field": source_field,
"field": actual_field_name,
"source_field": source_field,
"operator": insensitive_contains,
"value_encoder": string_encoder,
},
"{}__istartswith".format(field_name): {
"field": source_field,
"field": actual_field_name,
"source_field": source_field,
"operator": insensitive_starts_with,
"value_encoder": string_encoder,
},
"{}__iendswith".format(field_name): {
"field": source_field,
"field": actual_field_name,
"source_field": source_field,
"operator": insensitive_ends_with,
"value_encoder": string_encoder,
},
Expand Down
9 changes: 5 additions & 4 deletions tortoise/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,10 +283,10 @@ class Model(metaclass=ModelMeta):
# I don' like this here, but it makes autocompletion and static analysis much happier
_meta = MetaInfo(None)

def __init__(self, *args, _from_db: bool = False, **kwargs) -> None:
def __init__(self, *args, **kwargs) -> None:
# self._meta is a very common attribute lookup, lets cache it.
meta = self._meta
self._saved_in_db = _from_db or (meta.pk_attr in kwargs and meta.pk.generated)
self._saved_in_db = meta.pk_attr in kwargs and meta.pk.generated
self._init_lazy_fkm2m()

# Assign values and do type conversions
Expand All @@ -311,8 +311,9 @@ def _init_from_db(cls, **kwargs) -> MODEL_TYPE:
meta = self._meta

for key, value in kwargs.items():
if key in meta.fields:
setattr(self, key, meta.fields_map[key].to_python_value(value))
model_field = meta.fields_db_projection_reverse.get(key)
if model_field:
setattr(self, model_field, meta.fields_map[model_field].to_python_value(value))

return self

Expand Down
2 changes: 1 addition & 1 deletion tortoise/query_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def _process_filter_kwarg(model, key, value) -> Tuple[Criterion, Optional[Tuple[
if param.get("value_encoder")
else model._meta.db.executor_class._field_to_db(field_object, value, model)
)
criterion = param["operator"](getattr(table, param["field"]), encoded_value)
criterion = param["operator"](getattr(table, param["source_field"]), encoded_value)
return criterion, join


Expand Down
24 changes: 22 additions & 2 deletions tortoise/tests/models_schema_create.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@


class Tournament(Model):
id = fields.IntField(pk=True)
tid = fields.IntField(pk=True)
name = fields.TextField(description="Tournament name", index=True)
created = fields.DatetimeField(auto_now_add=True, description="Created */'`/* datetime")

Expand All @@ -23,7 +23,7 @@ class Event(Model):
participants = fields.ManyToManyField(
"models.Team",
related_name="events",
through="event_team",
through="teamevents",
description="How participants relate",
)
modified = fields.DatetimeField(auto_now=True)
Expand All @@ -36,6 +36,26 @@ class Meta:

class Team(Model):
name = fields.CharField(max_length=50, pk=True, description="The TEAM name (and PK)")
manager = fields.ForeignKeyField("models.Team", related_name="team_members", null=True)
talks_to = fields.ManyToManyField("models.Team", related_name="gets_talked_to")

class Meta:
table_description = "The TEAMS!"


class SourceFields(Model):
id = fields.IntField(pk=True, source_field="sometable_id")
chars = fields.CharField(max_length=255, source_field="some_chars_table", index=True)
fk = fields.ForeignKeyField(
"models.SourceFields", related_name="team_members", null=True, source_field="fk_sometable"
)
rel_to = fields.ManyToManyField(
"models.SourceFields",
related_name="rel_from",
through="sometable_self",
forward_key="sts_forward",
backward_key="backward_sts",
)

class Meta:
table = "sometable"
Loading

0 comments on commit 006be72

Please sign in to comment.