diff --git a/CHANGES.md b/CHANGES.md index be507b1..e9636e6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,23 @@ Changelog ========= -0.1.0 (Oct 7th 2015) +1.0.0 (Oct 16th 2016) +------------------ + +### Django 1.10 compatibility and some additional library features! + +- Added decorator for shard storage. +- Renamed `PostgresShardGeneratedIDField` to `PostgresShardGeneratedIDAutoField`. +- Added non-autoid `PostgresShardGeneratedIDField` that makes a separate call to +the database prior to saving. Good for statement based replication. Now you can +have more than one of these fields on a model. +- Fix `TableShardedIDField` to take a table name rather than model class so that +it doesn't give errors when reading the migrations file after deleting the table. +- Fix `showmigrations` to use the same database param as `migrate` and act on +all by default. + + +0.1.0 (Oct 7th 2016) ------------------ ### Django 1.10 compatibility and some additional library features! diff --git a/README.md b/README.md index 951f625..964b147 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ class ShardedCarIDs(TableStrategyModel): @model_config(sharded=True) class Car(models.Model): - id = TableShardedIDField(primary_key=True, source_table=ShardeCarIDs) + id = TableShardedIDField(primary_key=True, source_table_name='app.ShardeCarIDs') ignition_type = models.CharField(max_length=120) company = models.ForeignKey('companies.Company') diff --git a/django_sharding/management/commands/showmigrations.py b/django_sharding/management/commands/showmigrations.py new file mode 100644 index 0000000..944a313 --- /dev/null +++ b/django_sharding/management/commands/showmigrations.py @@ -0,0 +1,5 @@ +from django_sharding_library.management.commands.showmigrations import Command as ShowMigrationsCommand + + +class Command(ShowMigrationsCommand): + pass diff --git a/django_sharding_library/decorators.py b/django_sharding_library/decorators.py index 33d400d..7310971 100644 --- a/django_sharding_library/decorators.py +++ b/django_sharding_library/decorators.py @@ -7,12 +7,21 @@ from django_sharding_library.exceptions import NonExistentDatabaseException, ShardedModelInitializationException from django_sharding_library.manager import ShardManager -from django_sharding_library.fields import ShardedIDFieldMixin, PostgresShardGeneratedIDField +from django_sharding_library.fields import ShardedIDFieldMixin, BasePostgresShardGeneratedIDField from django_sharding_library.utils import register_migration_signal_for_model_receiver PRE_MIGRATION_DISPATCH_UID = "PRE_MIGRATE_FOR_MODEL_%s" +def shard_storage_config(shard_group='default', shared_field='shard'): + def configure(cls): + setattr(cls, 'django_sharding__shard_group', shard_group) + setattr(cls, 'django_sharding__shard_field', shared_field) + setattr(cls, 'django_sharding__stores_shard', True) + return cls + return configure + + def model_config(shard_group=None, database=None, sharded_by_field=None): """ A decorator for marking a model as being either sharded or stored on a @@ -33,7 +42,7 @@ def configure(cls): ) setattr(cls, 'django_sharding__database', database) - postgres_shard_id_fields = list(filter(lambda field: issubclass(type(field), PostgresShardGeneratedIDField), cls._meta.fields)) + postgres_shard_id_fields = list(filter(lambda field: issubclass(type(field), BasePostgresShardGeneratedIDField), cls._meta.fields)) if postgres_shard_id_fields: database_dicts = [settings.DATABASES[database]] if database else [db_settings for db, db_settings in iteritems(settings.DATABASES) if @@ -42,9 +51,10 @@ def configure(cls): raise ShardedModelInitializationException( 'You cannot use a PostgresShardGeneratedIDField on a non-Postgres database.') - register_migration_signal_for_model_receiver(apps.get_app_config(cls._meta.app_label), - PostgresShardGeneratedIDField.migration_receiver, - dispatch_uid=PRE_MIGRATION_DISPATCH_UID % cls._meta.app_label) + for field in postgres_shard_id_fields: + register_migration_signal_for_model_receiver(apps.get_app_config(cls._meta.app_label), + field.migration_receiver, + dispatch_uid=PRE_MIGRATION_DISPATCH_UID % cls._meta.app_label) if shard_group: sharded_fields = list(filter(lambda field: issubclass(type(field), ShardedIDFieldMixin), cls._meta.fields)) @@ -52,9 +62,9 @@ def configure(cls): raise ShardedModelInitializationException('All sharded models require a ShardedIDFieldMixin or a ' 'PostgresShardGeneratedIDField.') - if not list(filter(lambda field: field == cls._meta.pk, sharded_fields)) and not postgres_shard_id_fields: - raise ShardedModelInitializationException('All sharded models require the ShardedAutoIDField or ' - 'PostgresShardGeneratedIDFieldto be the primary key. Set ' + if not list(filter(lambda field: field == cls._meta.pk, sharded_fields + postgres_shard_id_fields)): + raise ShardedModelInitializationException('All sharded models require a ShardedAutoIDField or ' + 'PostgresShardGeneratedIDField to be the primary key. Set ' 'primary_key=True on the field.') if not callable(getattr(cls, 'get_shard', None)): diff --git a/django_sharding_library/exceptions.py b/django_sharding_library/exceptions.py index 2121883..44396fb 100644 --- a/django_sharding_library/exceptions.py +++ b/django_sharding_library/exceptions.py @@ -10,5 +10,9 @@ class InvalidMigrationException(DjangoShardingException): pass +class InvalidShowMigrationsException(DjangoShardingException): + pass + + class NonExistentDatabaseException(DjangoShardingException): pass diff --git a/django_sharding_library/fields.py b/django_sharding_library/fields.py index 2dcba80..d794f8f 100644 --- a/django_sharding_library/fields.py +++ b/django_sharding_library/fields.py @@ -3,7 +3,7 @@ from django.db.models import AutoField, CharField, ForeignKey, BigIntegerField, OneToOneField from django_sharding_library.constants import Backends -from django_sharding_library.utils import create_postgres_global_sequence, create_postgres_shard_id_function +from django_sharding_library.utils import create_postgres_global_sequence, create_postgres_shard_id_function, get_next_sharded_id try: from django.db.backends.postgresql.base import DatabaseWrapper as PostgresDatabaseWrapper @@ -59,19 +59,19 @@ def get_pk_value_on_save(self, instance): class TableShardedIDField(ShardedIDFieldMixin, BigAutoField): """ - An autoincrimenting field which takes a `source_table` as an argument in - order to generate unqiue ids for the sharded model. + An autoincrimenting field which takes a `source_table_name` as an argument in + order to generate unqiue ids for the sharded model. i.e. `app.model_name`. """ def __init__(self, *args, **kwargs): from django_sharding_library.id_generation_strategies import TableStrategy - kwargs['strategy'] = TableStrategy(backing_model=kwargs['source_table']) - setattr(self, 'source_table', kwargs['source_table']) - del kwargs['source_table'] + kwargs['strategy'] = TableStrategy(backing_model_name=kwargs['source_table_name']) + setattr(self, 'source_table_name', kwargs['source_table_name']) + del kwargs['source_table_name'] return super(TableShardedIDField, self).__init__(*args, **kwargs) def deconstruct(self): name, path, args, kwargs = super(TableShardedIDField, self).deconstruct() - kwargs['source_table'] = getattr(self, 'source_table') + kwargs['source_table_name'] = getattr(self, 'source_table_name') return name, path, args, kwargs @@ -169,25 +169,14 @@ class ShardForeignKeyStorageField(ShardForeignKeyStorageFieldMixin, ForeignKey): pass -class PostgresShardGeneratedIDField(AutoField): - """ - A field that uses a Postgres stored procedure to return an ID generated on the database. - """ - def db_type(self, connection, *args, **kwargs): +class BasePostgresShardGeneratedIDField(object): + + def __init__(self, *args, **kwargs): if not hasattr(settings, 'SHARD_EPOCH'): raise ValueError("PostgresShardGeneratedIDField requires a SHARD_EPOCH to be defined in your settings file.") - if connection.vendor == PostgresDatabaseWrapper.vendor: - return "bigint DEFAULT next_sharded_id()" - else: - return super(PostgresShardGeneratedIDField, self).db_type(connection) - - def get_internal_type(self): - return 'BigIntegerField' - - def rel_db_type(self, connection): - return BigIntegerField().db_type(connection=connection) + return super(BasePostgresShardGeneratedIDField, self).__init__(*args, **kwargs) @staticmethod def migration_receiver(*args, **kwargs): @@ -202,6 +191,51 @@ def migration_receiver(*args, **kwargs): create_postgres_shard_id_function(sequence_name, db_alias, shard_id) +class PostgresShardGeneratedIDAutoField(BasePostgresShardGeneratedIDField, BigAutoField): + """ + A field that uses a Postgres stored procedure to return an ID generated on the database. + """ + def db_type(self, connection, *args, **kwargs): + if connection.vendor == PostgresDatabaseWrapper.vendor: + return "bigint DEFAULT next_sharded_id()" + else: + return super(PostgresShardGeneratedIDAutoField, self).db_type(connection) + + +class PostgresShardGeneratedIDField(BasePostgresShardGeneratedIDField, BigIntegerField): + """ + A field that uses a Postgres stored procedure to return an ID generated on the database. + + Generates them prior to save with a seperate call to the DB. + """ + + def get_shard_from_id(self, instance_id): + group = getattr(self, 'django_sharding__shard_group', None) + shard_id_to_find = int(bin(instance_id)[-23:-10], 2) # We know where the shard id is stored in the PK's bits. + + # We can check the shard id from the PK against the shard ID in the databases config + for alias, db_settings in settings.DATABASES.items(): + if db_settings["SHARD_GROUP"] == group and db_settings["SHARD_ID"] == shard_id_to_find: + return alias + + return None # Return None if we could not determine the shard so we can fall through to the next shard grab attempt + + def get_pk_value_on_save(self, instance): + return self.generate_id(instance) + + def pre_save(self, model_instance, add): + if getattr(model_instance, self.attname, None) is not None: + return super(PostgresShardGeneratedIDField, self).pre_save(model_instance, add) + value = self.generate_id(model_instance) + setattr(model_instance, self.attname, value) + return value + + @staticmethod + def generate_id(instance): + shard = instance._state.db or instance.get_shard() + return get_next_sharded_id(shard) + + class PostgresShardForeignKey(ForeignKey): def db_type(self, connection): # The database column type of a ForeignKey is the column type diff --git a/django_sharding_library/id_generation_strategies.py b/django_sharding_library/id_generation_strategies.py index 2cd0694..f4d78ee 100644 --- a/django_sharding_library/id_generation_strategies.py +++ b/django_sharding_library/id_generation_strategies.py @@ -1,5 +1,6 @@ import uuid +from django.apps import apps from django.db import connections, transaction from django.utils.deconstruct import deconstructible @@ -24,33 +25,38 @@ class TableStrategy(BaseIDGenerationStrategy): Uses an autoincrement field, on a TableStrategyModel model `backing_model` to generate unique IDs. """ - def __init__(self, backing_model): - if not issubclass(backing_model, TableStrategyModel): - raise ValueError("Unsupported model used for generating IDs") - self.backing_model = backing_model + def __init__(self, backing_model_name): + self.backing_model_name = backing_model_name def get_next_id(self, database=None): """ Returns a new unique integer identifier for an object using an auto-incrimenting field in the database. """ + app_label = self.backing_model_name.split('.')[0] + app = apps.get_app_config(app_label) + backing_model = app.get_model(self.backing_model_name[len(app_label) + 1:]) + + if not issubclass(backing_model, TableStrategyModel): + raise ValueError("Unsupported model used for generating IDs") + from django.conf import settings - backing_table_db = getattr(self.backing_model, 'database', 'default') + backing_table_db = getattr(backing_model, 'database', 'default') if settings.DATABASES[backing_table_db]['ENGINE'] in Backends.MYSQL: with transaction.atomic(backing_table_db): cursor = connections[backing_table_db].cursor() sql = "REPLACE INTO `{0}` (`stub`) VALUES ({1})".format( - self.backing_model._meta.db_table, True + backing_model._meta.db_table, True ) cursor.execute(sql) if getattr(cursor.cursor.cursor, 'lastrowid', None): id = cursor.cursor.cursor.lastrowid else: - id = self.backing_model.objects.get(stub=True).id + id = backing_model.objects.get(stub=True).id else: with transaction.atomic(backing_table_db): - id = self.backing_model.objects.create(stub=None).id + id = backing_model.objects.create(stub=None).id return id diff --git a/django_sharding_library/management/commands/showmigrations.py b/django_sharding_library/management/commands/showmigrations.py new file mode 100644 index 0000000..ffe2a04 --- /dev/null +++ b/django_sharding_library/management/commands/showmigrations.py @@ -0,0 +1,33 @@ +from django.conf import settings +from django.core.management.commands.showmigrations import Command as ShowMigrationsCommand + +from django_sharding_library.exceptions import InvalidShowMigrationsException + + +class Command(ShowMigrationsCommand): + def handle(self, *args, **options): + if not options['database'] or options['database'] == 'all': + databases = self.get_all_but_replica_dbs() + elif options['database'] not in self.get_all_but_replica_dbs(): + raise InvalidShowMigrationsException('You must use showmigrations an existing non-primary DB.') + else: + databases = [options['database']] + + for database in databases: + options['database'] = database + # Writen in green text to stand out from the surrouding headings + if options['verbosity'] >= 1: + self.stdout.write(getattr(self.style, "MIGRATE_SUCCESS", getattr(self.style, "SUCCESS", lambda a: a))("\nDatabase: {}\n").format(database)) + super(Command, self).handle(*args, **options) + + def get_all_but_replica_dbs(self): + return list(filter( + lambda db: not settings.DATABASES[db].get('PRIMARY', None), + settings.DATABASES.keys() + )) + + def add_arguments(self, parser): + super(Command, self).add_arguments(parser) + parser._option_string_actions['--database'].default = None + parser._option_string_actions['--database'].help = u'Nominates a database to synchronize. Defaults to all databases.' + parser._option_string_actions['--database'].choices = ['all'] + self.get_all_but_replica_dbs() diff --git a/django_sharding_library/router.py b/django_sharding_library/router.py index 0f37eb4..7bf7486 100644 --- a/django_sharding_library/router.py +++ b/django_sharding_library/router.py @@ -1,7 +1,7 @@ from django.apps import apps from django.conf import settings -from django_sharding_library.fields import PostgresShardGeneratedIDField +from django_sharding_library.fields import BasePostgresShardGeneratedIDField from django_sharding_library.exceptions import DjangoShardingException, InvalidMigrationException from django_sharding_library.utils import ( is_model_class_on_database, @@ -69,7 +69,7 @@ def _get_shard(self, model, **hints): if sharded_by_field_id: shard = self.get_shard_for_id_field(model, sharded_by_field_id) - is_pk_postgres_generated_id_field = isinstance(getattr(model._meta, 'pk'), PostgresShardGeneratedIDField) + is_pk_postgres_generated_id_field = issubclass(type(getattr(model._meta, 'pk')), BasePostgresShardGeneratedIDField) lookup_pk = hints.get('exact_lookups', {}).get('pk') or hints.get('exact_lookups', {}).get('id') if shard is None and is_pk_postgres_generated_id_field and lookup_pk is not None: diff --git a/django_sharding_library/utils.py b/django_sharding_library/utils.py index a2b70c4..940c1d8 100644 --- a/django_sharding_library/utils.py +++ b/django_sharding_library/utils.py @@ -6,7 +6,6 @@ from django_sharding_library.exceptions import DjangoShardingException - def create_postgres_global_sequence(sequence_name, db_alias, reset_sequence=False): cursor = connections[db_alias].cursor() sid = transaction.savepoint(db_alias) @@ -110,3 +109,12 @@ def get_database_for_model_instance(instance): return instance.get_shard() raise DjangoShardingException("Unable to deduce datbase for model instance") + + +def get_next_sharded_id(shard): + cursor = connections[shard].cursor() + cursor.execute("SELECT next_sharded_id();") + generated_id = cursor.fetchone() + cursor.close() + + return generated_id[0] diff --git a/docs/components/OtherComponents.md b/docs/components/OtherComponents.md index 9bca987..1260f7d 100644 --- a/docs/components/OtherComponents.md +++ b/docs/components/OtherComponents.md @@ -106,19 +106,19 @@ As an example using the above mixin, one of the included fields uses a secondary class TableShardedIDField(ShardedIDFieldMixin, BigAutoField): """ - An autoincrimenting field which takes a `source_table` as an argument in + An autoincrimenting field which takes a `source_table_name` as an argument in order to generate unqiue ids for the sharded model. """ def __init__(self, *args, **kwargs): from django_sharding_library.id_generation_strategies import TableStrategy - kwargs['strategy'] = TableStrategy(backing_model=kwargs['source_table']) - setattr(self, 'source_table', kwargs['source_table']) - del kwargs['source_table'] + kwargs['strategy'] = TableStrategy(backing_model_name=kwargs['source_table_name']) + setattr(self, 'source_table_name', kwargs['source_table_name']) + del kwargs['source_table_name'] return super(TableShardedIDField, self).__init__(*args, **kwargs) def deconstruct(self): name, path, args, kwargs = super(TableShardedIDField, self).deconstruct() - kwargs['source_table'] = getattr(self, 'source_table') + kwargs['source_table_name'] = getattr(self, 'source_table_name') return name, path, args, kwargs ``` diff --git a/docs/usage/ShardingAModel.md b/docs/usage/ShardingAModel.md index c27a79f..de85839 100644 --- a/docs/usage/ShardingAModel.md +++ b/docs/usage/ShardingAModel.md @@ -2,7 +2,7 @@ ### Defining The Shard Key -Based on the earlier sections of the documentation, you need to choose a sharding function, strategy and ID generation strategy. +Based on the earlier sections of the documentation, you need to choose a sharding function, strategy and ID generation strategy. #### Storing The Shard On The Model With The Shard Key @@ -77,14 +77,14 @@ class ShardedCoolGuyModelIDs(TableStrategyModel): @model_config(shard_group='default', sharded_by_field='user_pk') class CoolGuyShardedModel(models.Model): - id = TableShardedIDField(primary_key=True, source_table=ShardedCoolGuyModelIDs) + id = TableShardedIDField(primary_key=True, source_table_name='app.ShardedCoolGuyModelIDs') cool_guy_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() def get_shard(self): from django.contrib.auth import get_user_model return get_user_model().objects.get(pk=self.user_pk).shard - + @staticmethod def get_shard_from_id(user_pk): from django.contrib.auth import get_user_model @@ -133,4 +133,4 @@ SHARD_EPOCH=int(time.mktime(datetime(2016, 1, 1).timetuple()) * 1000) ``` 3. When you are editing your DATABASES settings, the order of the shards MUST be maintained. If you add a new shard, it needs to be added to the end of the list of databases, not to the beginning or middle. -4. There is a maximum number of logical shards supported by this field. You can only have up to 8191 logical shards: if you try to go beyond, you will get duplicate IDs between your shards. Do not try to add more than 8191 shards. If you need more than that, I recommend you choose one of the other ID generation strategies. \ No newline at end of file +4. There is a maximum number of logical shards supported by this field. You can only have up to 8191 logical shards: if you try to go beyond, you will get duplicate IDs between your shards. Do not try to add more than 8191 shards. If you need more than that, I recommend you choose one of the other ID generation strategies. diff --git a/setup.py b/setup.py index 34835a9..e212949 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '0.1.0' +version = '1.0.0' setup( diff --git a/tests/models.py b/tests/models.py index 20eac2e..f806433 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,8 +1,13 @@ from django.contrib.auth.models import AbstractUser from django.db import models from django.conf import settings -from django_sharding_library.decorators import model_config -from django_sharding_library.fields import TableShardedIDField, ShardForeignKeyStorageField, PostgresShardGeneratedIDField +from django_sharding_library.decorators import model_config, shard_storage_config +from django_sharding_library.fields import ( + TableShardedIDField, + ShardForeignKeyStorageField, + PostgresShardGeneratedIDAutoField, + PostgresShardGeneratedIDField +) from django_sharding_library.models import ShardedByMixin, ShardStorageModel, TableStrategyModel from django_sharding_library.constants import Backends @@ -16,15 +21,16 @@ class ShardedModelIDs(TableStrategyModel): # An implimentation of the extension of a the Django user to add # the mixin provided in order to save the shard on the user. +@shard_storage_config() class User(AbstractUser, ShardedByMixin): - django_sharding__shard_group = 'default' + pass # An implimentation of the extension of a the Django user to add # the mixin provided in order to save the shard on the user. +@shard_storage_config(shard_group='postgres') class PostgresShardUser(AbstractUser, ShardedByMixin): - shard_group = 'postgres' - django_sharding__shard_group = 'postgres' + pass # A model for use with a sharded model to generate pk's using @@ -44,7 +50,7 @@ class ShardedTestModelIDs(TableStrategyModel): @model_config(shard_group='default', sharded_by_field='user_pk') class TestModel(models.Model): - id = TableShardedIDField(primary_key=True, source_table=ShardedTestModelIDs) + id = TableShardedIDField(primary_key=True, source_table_name='tests.ShardedTestModelIDs') random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() @@ -61,7 +67,7 @@ def get_shard_from_id(user_pk): @model_config(database='default') class UnshardedTestModel(models.Model): - id = TableShardedIDField(primary_key=True, source_table=ShardedTestModelIDs) + id = TableShardedIDField(primary_key=True, source_table_name='tests.ShardedTestModelIDs') random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() @@ -89,11 +95,30 @@ class PostgresCustomIDModelBackupField(TableStrategyModel): @model_config(shard_group="postgres", sharded_by_field="user_pk") +class PostgresCustomAutoIDModel(models.Model): + if settings.DATABASES['default']['ENGINE'] in Backends.POSTGRES: + id = PostgresShardGeneratedIDAutoField(primary_key=True) + else: + id = TableShardedIDField(primary_key=True, source_table_name='tests.PostgresCustomIDModelBackupField') + random_string = models.CharField(max_length=120) + user_pk = models.PositiveIntegerField() + + def get_shard(self): + return PostgresShardUser.objects.get(pk=self.user_pk).shard + + @staticmethod + def get_shard_from_id(user_pk): + return PostgresShardUser.objects.get(pk=user_pk).shard + + +@model_config(shard_group="postgres") class PostgresCustomIDModel(models.Model): if settings.DATABASES['default']['ENGINE'] in Backends.POSTGRES: id = PostgresShardGeneratedIDField(primary_key=True) + some_field = PostgresShardGeneratedIDField() else: - id = TableShardedIDField(primary_key=True, source_table=PostgresCustomIDModelBackupField) + id = TableShardedIDField(primary_key=True, source_table_name='tests.PostgresCustomIDModelBackupField') + some_field = models.PositiveIntegerField() random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() diff --git a/tests/test_decorators.py b/tests/test_decorators.py index a4544b3..fb7e349 100644 --- a/tests/test_decorators.py +++ b/tests/test_decorators.py @@ -1,24 +1,24 @@ +import unittest + +from django.conf import settings from django.db import models from django_sharding_library.id_generation_strategies import TableStrategyModel from django_sharding_library.decorators import model_config from django_sharding_library.exceptions import NonExistentDatabaseException, ShardedModelInitializationException -from django_sharding_library.fields import TableShardedIDField +from django_sharding_library.fields import PostgresShardGeneratedIDField, TableShardedIDField from django.test import TestCase from django_sharding_library.manager import ShardManager +from django_sharding_library.constants import Backends -class ModelConfigDecoratorTestCase(TestCase): - def setUp(self): - class ShardedDecoratorTestModelIDs(TableStrategyModel): - pass - self.id_table = ShardedDecoratorTestModelIDs +class ModelConfigDecoratorTestCase(TestCase): def test_model_cannot_be_both_sharded_and_marked_for_a_specific_db(self): with self.assertRaises(ShardedModelInitializationException): @model_config(shard_group='default', database='default') class TestModelTwo(models.Model): - id = TableShardedIDField(primary_key=True, source_table=self.id_table) + id = TableShardedIDField(primary_key=True, source_table_name="blah") random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() @@ -29,7 +29,7 @@ def test_sharded_model_requires_a_get_shard_method(self): with self.assertRaises(ShardedModelInitializationException): @model_config(shard_group='default') class TestModelTwo(models.Model): - id = TableShardedIDField(primary_key=True, source_table=self.id_table) + id = TableShardedIDField(primary_key=True, source_table_name="blah") random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() @@ -37,7 +37,7 @@ def test_sharded_id_field_must_be_primary_key(self): with self.assertRaises(ShardedModelInitializationException): @model_config(shard_group='default') class TestModelTwo(models.Model): - id = TableShardedIDField(source_table=self.id_table) + id = TableShardedIDField(source_table_name="blah") random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField(primary_key=True) @@ -58,7 +58,22 @@ def get_shard(self): def test_puts_shard_group_on_the_model_class(self): @model_config(shard_group='testing') class TestModelThree(models.Model): - id = TableShardedIDField(source_table=self.id_table, primary_key=True) + id = TableShardedIDField(source_table_name="blah", primary_key=True) + random_string = models.CharField(max_length=120) + user_pk = models.PositiveIntegerField() + + def get_shard(self): + from django.contrib.auth import get_user_model + return get_user_model().objects.get(pk=self.user_pk).shard + + self.assertEqual(getattr(TestModelThree, 'django_sharding__shard_group', None), 'testing') + + @unittest.skipIf(settings.DATABASES['default']['ENGINE'] not in Backends.POSTGRES, "Not a postgres backend") + def test_two_postgres_sharded_id_generator_fields(self): + @model_config(shard_group='testing') + class TestModelThree(models.Model): + id = PostgresShardGeneratedIDField(primary_key=True) + something = PostgresShardGeneratedIDField() random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() @@ -92,7 +107,7 @@ def test_abstract_model_with_defined_manager_raises_exception_if_not_instance_of with self.assertRaises(ShardedModelInitializationException): @model_config(shard_group='default', sharded_by_field="user_pk") class TestModelOne(models.Model): - id = TableShardedIDField(primary_key=True, source_table=self.id_table) + id = TableShardedIDField(primary_key=True, source_table_name="blah") random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() @@ -111,7 +126,7 @@ def get_shard_from_id(id): # Manager is not defined, should NOT raise an exception @model_config(shard_group='default', sharded_by_field="user_pk") class TestModelTwo(models.Model): - id = TableShardedIDField(primary_key=True, source_table=self.id_table) + id = TableShardedIDField(primary_key=True, source_table_name="blah") random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() @@ -128,7 +143,7 @@ def get_shard_from_id(id): # Manager is defines but is a shardmanager, should not raise an exception @model_config(shard_group='default', sharded_by_field="user_pk") class TestModelThree(models.Model): - id = TableShardedIDField(primary_key=True, source_table=self.id_table) + id = TableShardedIDField(primary_key=True, source_table_name="blah") random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() @@ -148,7 +163,7 @@ def test_decorator_raises_exception_when_sharded_by_field_is_defined_with_no_get with self.assertRaises(ShardedModelInitializationException): @model_config(shard_group='default', sharded_by_field="user_pk") class TestModelOne(models.Model): - id = TableShardedIDField(primary_key=True, source_table=self.id_table) + id = TableShardedIDField(primary_key=True, source_table_name="blah") random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() @@ -162,7 +177,7 @@ class CustomManager(models.Manager): with self.assertRaises(ShardedModelInitializationException): @model_config(shard_group='default', sharded_by_field="user_pk") class TestModelOne(models.Model): - id = TableShardedIDField(primary_key=True, source_table=self.id_table) + id = TableShardedIDField(primary_key=True, source_table_name="blah") random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() @@ -179,7 +194,7 @@ def test_decorator_raises_exception_when_no_arguments_passed_in(self): with self.assertRaises(ShardedModelInitializationException): @model_config() class TestModelOne(models.Model): - id = TableShardedIDField(primary_key=True, source_table=self.id_table) + id = TableShardedIDField(primary_key=True, source_table_name="blah") random_string = models.CharField(max_length=120) user_pk = models.PositiveIntegerField() diff --git a/tests/test_fields.py b/tests/test_fields.py index 4873f2d..e51afc0 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -24,6 +24,7 @@ ShardedTestModelIDs, TestModel, ShardStorageTable, + PostgresCustomAutoIDModel, PostgresCustomIDModel, PostgresShardUser ) @@ -198,7 +199,7 @@ def test_check_shard_id_function(self): @unittest.skipIf(settings.DATABASES['default']['ENGINE'] not in Backends.POSTGRES, "Not a postgres backend") def test_check_shard_id_returns_with_model_save(self): user = PostgresShardUser.objects.create_user(username='username', password='pwassword', email='test@example.com') - created_model = PostgresCustomIDModel.objects.create(random_string='Test String', user_pk=user.id) + created_model = PostgresCustomAutoIDModel.objects.create(random_string='Test String', user_pk=user.id) self.assertTrue(getattr(created_model, 'id')) # Same as above, lets create an id that would have been made 10 seconds ago and make sure the one that was @@ -207,3 +208,41 @@ def test_check_shard_id_returns_with_model_save(self): lowest_id |= 0 << 10 lowest_id |= 1 self.assertGreater(created_model.id, lowest_id) + + @unittest.skipIf(settings.DATABASES['default']['ENGINE'] not in Backends.POSTGRES, "Not a postgres backend") + def test_check_shard_id_generated_prior_to_model_save(self): + user = PostgresShardUser.objects.create_user(username='username', password='pwassword', email='test@example.com') + + shard_id = settings.DATABASES[user.shard]['SHARD_ID'] + seq_id = 800 + sharded_instance_id = ((settings.SHARD_EPOCH + 10) - settings.SHARD_EPOCH) << 23 + sharded_instance_id |= (shard_id << 10) + sharded_instance_id |= (seq_id) + + with patch('django_sharding_library.fields.get_next_sharded_id', return_value=sharded_instance_id): + created_model = PostgresCustomIDModel.objects.using(user.shard).create(random_string='Test String', user_pk=user.id) + + self.assertEqual(created_model.id, sharded_instance_id) + + @unittest.skipIf(settings.DATABASES['default']['ENGINE'] not in Backends.POSTGRES, "Not a postgres backend") + def test_check_shard_id_generated_prior_to_model_save_ordered(self): + user = PostgresShardUser.objects.create_user(username='username', password='pwassword', email='test@example.com') + created_model = PostgresCustomIDModel.objects.using(user.shard).create(random_string='Test String', user_pk=user.id) + self.assertTrue(getattr(created_model, 'id')) + + # Same as above, lets create an id that would have been made 10 seconds ago and make sure the one that was + # created and returned is larger + lowest_id = int(time.mktime(datetime.now().timetuple()) * 1000) - settings.SHARD_EPOCH - 10000 << 23 + lowest_id |= 0 << 10 + lowest_id |= 1 + self.assertGreater(created_model.id, lowest_id) + + @unittest.skipIf(settings.DATABASES['default']['ENGINE'] not in Backends.POSTGRES, "Not a postgres backend") + def test_deconstruct_shard_from_id(self): + user = PostgresShardUser.objects.create_user(username='username', password='pwassword', email='test@example.com') + created_model = PostgresCustomIDModel.objects.using(user.shard).create(random_string='Test String', user_pk=user.id) + self.assertTrue(getattr(created_model, 'id')) + + instance_id = created_model.id + shard_id = int(bin(instance_id)[-23:-10], 2) + self.assertEqual(shard_id, settings.DATABASES[user.shard]['SHARD_ID']) diff --git a/tests/test_id_generation_strategies.py b/tests/test_id_generation_strategies.py index 287fe38..71d025d 100644 --- a/tests/test_id_generation_strategies.py +++ b/tests/test_id_generation_strategies.py @@ -12,12 +12,12 @@ class TableStrategyIDGenerationTestCase(TestCase): def test_returns_unique_values(self): - sut = TableStrategy(ShardedModelIDs) + sut = TableStrategy('tests.ShardedModelIDs') ids = [sut.get_next_id() for i in xrange(100)] self.assertEqual(ids, list(set(ids))) def test_largest_value_stored_in_db(self): - sut = TableStrategy(ShardedModelIDs) + sut = TableStrategy('tests.ShardedModelIDs') for i in xrange(100): id = sut.get_next_id() self.assertEqual(ShardedModelIDs.objects.latest('pk').pk, id) diff --git a/tests/test_router.py b/tests/test_router.py index bf8c994..d3b7852 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -6,11 +6,11 @@ from django.contrib.auth.models import Group from django.test import TransactionTestCase -from tests.models import TestModel, ShardedTestModelIDs, PostgresCustomIDModel, PostgresShardUser +from tests.models import TestModel, ShardedTestModelIDs, PostgresCustomAutoIDModel, PostgresShardUser from django_sharding_library.exceptions import InvalidMigrationException from django_sharding_library.router import ShardedRouter from django_sharding_library.routing_read_strategies import BaseRoutingStrategy -from django_sharding_library.fields import PostgresShardGeneratedIDField +from django_sharding_library.fields import PostgresShardGeneratedIDAutoField from django_sharding_library.constants import Backends from django_sharding_library.manager import ShardManager @@ -518,20 +518,20 @@ def setUp(self): @unittest.skipIf(settings.DATABASES['default']['ENGINE'] not in Backends.POSTGRES, "Not a postgres backend") def test_postgres_sharded_id_can_be_queried_without_using_and_without_sharded_by(self): - created_model = PostgresCustomIDModel.objects.create(random_string='Test String', user_pk=self.user.id) + created_model = PostgresCustomAutoIDModel.objects.create(random_string='Test String', user_pk=self.user.id) self.assertTrue(getattr(created_model, 'id')) - self.assertTrue(isinstance(PostgresCustomIDModel._meta.pk, PostgresShardGeneratedIDField)) + self.assertTrue(isinstance(PostgresCustomAutoIDModel._meta.pk, PostgresShardGeneratedIDAutoField)) - self.assertTrue(isinstance(PostgresCustomIDModel.objects, ShardManager)) + self.assertTrue(isinstance(PostgresCustomAutoIDModel.objects, ShardManager)) - instance = PostgresCustomIDModel.objects.get(id=created_model.id) + instance = PostgresCustomAutoIDModel.objects.get(id=created_model.id) self.assertEqual(created_model._state.db, instance._state.db) - instance = PostgresCustomIDModel.objects.get(pk=created_model.id) + instance = PostgresCustomAutoIDModel.objects.get(pk=created_model.id) self.assertEqual(created_model._state.db, instance._state.db) @unittest.skipIf(settings.DATABASES['default']['ENGINE'] not in Backends.POSTGRES, "Not a postgres backend") def test_shard_extracted_correctly(self): - created_model = PostgresCustomIDModel.objects.create(random_string='Test String', user_pk=self.user.pk) - self.assertEqual(self.user.shard, self.sut.get_shard_for_postgres_pk_field(PostgresCustomIDModel, created_model.id)) + created_model = PostgresCustomAutoIDModel.objects.create(random_string='Test String', user_pk=self.user.pk) + self.assertEqual(self.user.shard, self.sut.get_shard_for_postgres_pk_field(PostgresCustomAutoIDModel, created_model.id)) diff --git a/tests/test_showmigrations_command.py b/tests/test_showmigrations_command.py new file mode 100644 index 0000000..df79b12 --- /dev/null +++ b/tests/test_showmigrations_command.py @@ -0,0 +1,39 @@ +from django.core.management import call_command +from django.test import TestCase +from mock import patch + +from django_sharding_library.exceptions import InvalidShowMigrationsException + + +@patch('django.core.management.commands.showmigrations.Command.handle') +class MigrationCommandTestCase(TestCase): + + def test_defauls_migrates_all_primary_dbs(self, mock_migrate_command): + call_command('showmigrations', verbosity=0) + databases_migrated = [call[1].get('database') for call in mock_migrate_command.call_args_list] + expected_migrated_databases = ['app_shard_001', 'app_shard_002', 'app_shard_003', 'app_shard_004', 'default'] + + self.assertEqual(sorted(databases_migrated), expected_migrated_databases) + + def test_all_option_added_to_databases(self, mock_migrate_command): + call_command('showmigrations', database='all', verbosity=0) + databases_migrated = [call[1].get('database') for call in mock_migrate_command.call_args_list] + expected_migrated_databases = ['app_shard_001', 'app_shard_002', 'app_shard_003', 'app_shard_004', 'default'] + + self.assertEqual(sorted(databases_migrated), expected_migrated_databases) + + def test_migrate_single_db(self, mock_migrate_command): + call_command('showmigrations', database='default', verbosity=0) + databases_migrated = [call[1].get('database') for call in mock_migrate_command.call_args_list] + expected_migrated_databases = ['default'] + + self.assertEqual(sorted(databases_migrated), expected_migrated_databases) + + def test_migrate_replica_raises_exception(self, mock_migrate_command): + with self.assertRaises(InvalidShowMigrationsException): + call_command('showmigrations', database='app_shard_001_replica_001', verbosity=0) + + databases_migrated = [] + expected_migrated_databases = [] + + self.assertEqual(databases_migrated, expected_migrated_databases)