From 631c4a0d05d866ed3aaf8ab508eaf394b9eda284 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Tue, 25 Aug 2020 15:21:43 +0200 Subject: [PATCH 1/4] refactor(api): switch Python image from buster to Alpine --- api/Dockerfile | 23 +++++++++-------------- api/cronhook/start-cron.sh | 2 +- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 8e40a74f8..f5a752de7 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,22 +1,17 @@ -FROM python:3.8-buster - -RUN apt-get update && apt-get install -y \ - # updating this list may require updating /README.md - gcc \ - gettext \ - default-mysql-client default-libmysqlclient-dev \ - libmemcached-dev \ - postgresql-client libpq-dev \ - sqlite3 \ - cron \ - --no-install-recommends && apt-get clean && rm -rf /var/lib/apt/lists/* +FROM python:3.8-alpine + +RUN apk add --no-cache bash dcron sqlite RUN mkdir /usr/src/app WORKDIR /usr/src/app +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +ENV PIP_NO_CACHE_DIR=1 + COPY requirements.txt /usr/src/app/ -RUN pip install --no-cache-dir --upgrade pip -RUN pip install -r requirements.txt && rm -rf /root/.cache/ +RUN apk add --no-cache gcc freetype-dev libffi-dev musl-dev libmemcached-dev mariadb-connector-c-dev jpeg-dev zlib-dev \ + && pip install -r requirements.txt \ + && apk --no-cache del gcc RUN pip freeze RUN mkdir /root/cronhook diff --git a/api/cronhook/start-cron.sh b/api/cronhook/start-cron.sh index 66aa5699b..13ff283c1 100644 --- a/api/cronhook/start-cron.sh +++ b/api/cronhook/start-cron.sh @@ -3,5 +3,5 @@ printenv >> /etc/environment touch /var/log/cron.log -cron +crond -b -L /var/log/cron.log tail -F -v /var/log/cron.log From 557b01ea6cd53493162917ed68dc713f313fa6f6 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Thu, 20 Aug 2020 18:18:02 +0200 Subject: [PATCH 2/4] BREAKING chore(api): squash all migrations --- .../migrations/0001_initial_squashed.py | 109 --------- .../migrations/0001_initial_squashed_again.py | 210 ++++++++++++++++++ .../0002_lowercase_domains_and_subnames.py | 50 ----- api/desecapi/migrations/0003_validation.py | 30 --- .../migrations/0004_domain_minimum_ttl.py | 23 -- .../migrations/0005_user_model_cleanup.py | 21 -- .../migrations/0006_authenticated_actions.py | 79 ------- ...0007_remove_user_registration_remote_ip.py | 17 -- api/desecapi/migrations/0008_captcha.py | 24 -- .../0009_domain_minimum_ttl_default.py | 20 -- .../0010_hash_tokens_and_switch_to_uuid.py | 52 ----- .../migrations/0011_user_id_to_uuid.py | 88 -------- .../migrations/0012_volatile_donations.py | 17 -- .../migrations/0013_token_last_used.py | 45 ---- api/desecapi/migrations/0014_rrset_touched.py | 20 -- .../migrations/0015_rrset_touched_auto_now.py | 24 -- .../migrations/0016_domain_renewal.py | 62 ------ 17 files changed, 210 insertions(+), 681 deletions(-) delete mode 100644 api/desecapi/migrations/0001_initial_squashed.py create mode 100644 api/desecapi/migrations/0001_initial_squashed_again.py delete mode 100644 api/desecapi/migrations/0002_lowercase_domains_and_subnames.py delete mode 100644 api/desecapi/migrations/0003_validation.py delete mode 100644 api/desecapi/migrations/0004_domain_minimum_ttl.py delete mode 100644 api/desecapi/migrations/0005_user_model_cleanup.py delete mode 100644 api/desecapi/migrations/0006_authenticated_actions.py delete mode 100644 api/desecapi/migrations/0007_remove_user_registration_remote_ip.py delete mode 100644 api/desecapi/migrations/0008_captcha.py delete mode 100644 api/desecapi/migrations/0009_domain_minimum_ttl_default.py delete mode 100644 api/desecapi/migrations/0010_hash_tokens_and_switch_to_uuid.py delete mode 100644 api/desecapi/migrations/0011_user_id_to_uuid.py delete mode 100644 api/desecapi/migrations/0012_volatile_donations.py delete mode 100644 api/desecapi/migrations/0013_token_last_used.py delete mode 100644 api/desecapi/migrations/0014_rrset_touched.py delete mode 100644 api/desecapi/migrations/0015_rrset_touched_auto_now.py delete mode 100644 api/desecapi/migrations/0016_domain_renewal.py diff --git a/api/desecapi/migrations/0001_initial_squashed.py b/api/desecapi/migrations/0001_initial_squashed.py deleted file mode 100644 index aa13f074f..000000000 --- a/api/desecapi/migrations/0001_initial_squashed.py +++ /dev/null @@ -1,109 +0,0 @@ -# Generated by Django 2.2 on 2019-04-26 15:13 - -import desecapi.models -from django.conf import settings -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion -import uuid - - -class Migration(migrations.Migration): - - replaces = [('desecapi', '0001_initial'), ('desecapi', '0002_donation'), ('desecapi', '0003_auto_20151008_1023'), ('desecapi', '0004_remove_donation_rip'), ('desecapi', '0005_auto_20151008_1042'), ('desecapi', '0006_auto_20151018_1234'), ('desecapi', '0007_domain_updated'), ('desecapi', '0008_django_update_1-10'), ('desecapi', '0009_auto_20161201_1548'), ('desecapi', '0010_auto_20161219_1242'), ('desecapi', '0011_user_limit_domains'), ('desecapi', '0012_move_dyn_flag'), ('desecapi', '0013_acme_challenge'), ('desecapi', '0014_ip_validation'), ('desecapi', '0015_rrset'), ('desecapi', '0016_dyn_flag_default'), ('desecapi', '0017_rr_model'), ('desecapi', '0018_prune_domain_fields'), ('desecapi', '0019_rrset_uuid'), ('desecapi', '0020_user_locked'), ('desecapi', '0021_tokens'), ('desecapi', '0022_domain_published')] - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='User', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('password', models.CharField(max_length=128, verbose_name='password')), - ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), - ('email', models.EmailField(max_length=191, unique=True, verbose_name='email address')), - ('is_active', models.BooleanField(default=True)), - ('is_admin', models.BooleanField(default=False)), - ('registration_remote_ip', models.CharField(blank=True, max_length=1024)), - ('created', models.DateTimeField(auto_now_add=True)), - ('limit_domains', models.IntegerField(blank=True, default=5, null=True)), - ('dyn', models.BooleanField(default=False)), - ('locked', models.DateTimeField(blank=True, null=True)), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='Donation', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(default=desecapi.models.get_default_value_created)), - ('name', models.CharField(max_length=255)), - ('iban', models.CharField(max_length=34)), - ('bic', models.CharField(max_length=11)), - ('amount', models.DecimalField(decimal_places=2, max_digits=8)), - ('message', models.CharField(blank=True, max_length=255)), - ('due', models.DateTimeField(default=desecapi.models.get_default_value_due)), - ('mref', models.CharField(default=desecapi.models.get_default_value_mref, max_length=32)), - ('email', models.EmailField(blank=True, max_length=255)), - ], - options={ - 'ordering': ('created',), - }, - ), - migrations.CreateModel( - name='Domain', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True)), - ('name', models.CharField(max_length=191, unique=True)), - ('owner', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='domains', to=settings.AUTH_USER_MODEL)), - ('published', models.DateTimeField(null=True)), - ], - options={ - 'ordering': ('created',), - }, - ), - migrations.CreateModel( - name='RRset', - fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('created', models.DateTimeField(auto_now_add=True)), - ('updated', models.DateTimeField(null=True)), - ('subname', models.CharField(blank=True, max_length=178)), - ('type', models.CharField(max_length=10, validators=[desecapi.models.validate_upper])), - ('ttl', models.PositiveIntegerField(validators=[django.core.validators.MinValueValidator(1)])), - ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='desecapi.Domain')), - ], - options={ - 'unique_together': {('domain', 'subname', 'type')}, - }, - ), - migrations.CreateModel( - name='RR', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True)), - ('content', models.CharField(max_length=4092)), - ('rrset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='records', to='desecapi.RRset')), - ], - ), - migrations.CreateModel( - name='Token', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')), - ('key', models.CharField(db_index=True, max_length=40, unique=True, verbose_name='Key')), - ('name', models.CharField(default='', max_length=64, verbose_name='Name')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='auth_tokens', to=settings.AUTH_USER_MODEL, verbose_name='User')), - ('user_specific_id', models.BigIntegerField(verbose_name='User-Specific ID')), - ], - options={ - 'unique_together': {('user', 'user_specific_id')}, - }, - ), - ] diff --git a/api/desecapi/migrations/0001_initial_squashed_again.py b/api/desecapi/migrations/0001_initial_squashed_again.py new file mode 100644 index 000000000..608edaedf --- /dev/null +++ b/api/desecapi/migrations/0001_initial_squashed_again.py @@ -0,0 +1,210 @@ +# Generated by Django 3.1 on 2020-08-25 14:54 + +import desecapi.models +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import re +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='User', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('email', models.EmailField(max_length=191, unique=True, verbose_name='email address')), + ('is_active', models.BooleanField(default=True)), + ('is_admin', models.BooleanField(default=False)), + ('created', models.DateTimeField(auto_now_add=True)), + ('limit_domains', models.IntegerField(blank=True, default=5, null=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Domain', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('name', models.CharField(max_length=191, unique=True, validators=[desecapi.models.validate_lower, django.core.validators.RegexValidator(code='invalid_domain_name', flags=re.RegexFlag['IGNORECASE'], message='Domain names must be labels separated by dots. Labels may consist of up to 63 letters, digits, hyphens, and underscores. The last label may not contain an underscore.', regex='^(([a-z0-9_-]{1,63})\\.)*[a-z0-9-]{1,63}$')])), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='domains', to=settings.AUTH_USER_MODEL)), + ('published', models.DateTimeField(blank=True, null=True)), + ('minimum_ttl', models.PositiveIntegerField(default=desecapi.models.get_minimum_ttl_default)), + ('renewal_changed', models.DateTimeField(auto_now_add=True)), + ('renewal_state', models.IntegerField(choices=[(1, 'Fresh'), (2, 'Notified'), (3, 'Warned')], default=1)), + ], + options={ + 'ordering': ('created',), + }, + ), + migrations.CreateModel( + name='RRset', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True)), + ('touched', models.DateTimeField(auto_now=True)), + ('subname', models.CharField(blank=True, max_length=178, validators=[desecapi.models.validate_lower, django.core.validators.RegexValidator(code='invalid_subname', message="Subname can only use (lowercase) a-z, 0-9, ., -, and _, may start with a '*.', or just be '*'.", regex='^([*]|(([*][.])?[a-z0-9_.-]*))$')])), + ('type', models.CharField(max_length=10, validators=[desecapi.models.validate_upper, django.core.validators.RegexValidator(code='invalid_type', message='Type must be uppercase alphanumeric and start with a letter.', regex='^[A-Z][A-Z0-9]*$')])), + ('ttl', models.PositiveIntegerField()), + ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='desecapi.domain')), + ], + options={ + 'unique_together': {('domain', 'subname', 'type')}, + }, + ), + migrations.CreateModel( + name='AuthenticatedAction', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ], + options={ + 'managed': False, + }, + ), + migrations.CreateModel( + name='AuthenticatedUserAction', + fields=[ + ('authenticatedaction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.authenticatedaction')), + ], + options={ + 'managed': False, + }, + bases=('desecapi.authenticatedaction',), + ), + migrations.CreateModel( + name='AuthenticatedDeleteUserAction', + fields=[ + ('authenticateduseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.authenticateduseraction')), + ], + options={ + 'managed': False, + }, + bases=('desecapi.authenticateduseraction',), + ), + migrations.CreateModel( + name='AuthenticatedResetPasswordUserAction', + fields=[ + ('authenticateduseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.authenticateduseraction')), + ('new_password', models.CharField(max_length=128)), + ], + options={ + 'managed': False, + }, + bases=('desecapi.authenticateduseraction',), + ), + migrations.CreateModel( + name='Captcha', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True)), + ('content', models.CharField(default=desecapi.models.captcha_default_content, max_length=24)), + ], + ), + migrations.CreateModel( + name='Token', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')), + ('key', models.CharField(db_index=True, max_length=128, unique=True, verbose_name='Key')), + ('name', models.CharField(blank=True, max_length=64, verbose_name='Name')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='auth_tokens', to=settings.AUTH_USER_MODEL, verbose_name='User')), + ('last_used', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'verbose_name': 'Token', + 'verbose_name_plural': 'Tokens', + }, + ), + migrations.CreateModel( + name='RR', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('content', models.CharField(max_length=500)), + ('rrset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='records', to='desecapi.rrset')), + ], + ), + migrations.CreateModel( + name='AuthenticatedActivateUserAction', + fields=[ + ('authenticateduseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.authenticateduseraction')), + ('domain', models.CharField(max_length=191)), + ], + options={ + 'managed': False, + }, + bases=('desecapi.authenticateduseraction',), + ), + migrations.CreateModel( + name='AuthenticatedChangeEmailUserAction', + fields=[ + ('authenticateduseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.authenticateduseraction')), + ('new_email', models.EmailField(max_length=254)), + ], + options={ + 'managed': False, + }, + bases=('desecapi.authenticateduseraction',), + ), + migrations.CreateModel( + name='AuthenticatedBasicUserAction', + fields=[ + ('authenticatedaction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.authenticatedaction')), + ], + options={ + 'managed': False, + }, + bases=('desecapi.authenticatedaction',), + ), + migrations.CreateModel( + name='AuthenticatedDomainBasicUserAction', + fields=[ + ('authenticatedbasicuseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.authenticatedbasicuseraction')), + ], + options={ + 'managed': False, + }, + bases=('desecapi.authenticatedbasicuseraction',), + ), + migrations.CreateModel( + name='AuthenticatedRenewDomainBasicUserAction', + fields=[ + ('authenticateddomainbasicuseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.authenticateddomainbasicuseraction')), + ], + options={ + 'managed': False, + }, + bases=('desecapi.authenticateddomainbasicuseraction',), + ), + migrations.CreateModel( + name='Donation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(default=desecapi.models.get_default_value_created)), + ('name', models.CharField(max_length=255)), + ('iban', models.CharField(max_length=34)), + ('bic', models.CharField(max_length=11)), + ('amount', models.DecimalField(decimal_places=2, max_digits=8)), + ('message', models.CharField(blank=True, max_length=255)), + ('due', models.DateTimeField(default=desecapi.models.get_default_value_due)), + ('mref', models.CharField(default=desecapi.models.get_default_value_mref, max_length=32)), + ('email', models.EmailField(blank=True, max_length=255)), + ], + options={ + 'ordering': ('created',), + 'managed': False, + }, + ), + ] diff --git a/api/desecapi/migrations/0002_lowercase_domains_and_subnames.py b/api/desecapi/migrations/0002_lowercase_domains_and_subnames.py deleted file mode 100644 index d54efdfb5..000000000 --- a/api/desecapi/migrations/0002_lowercase_domains_and_subnames.py +++ /dev/null @@ -1,50 +0,0 @@ -import desecapi.models -import django.core.validators -from django.db import migrations, models - - -def lowercase_names(apps, schema_editor): - # Domains - Domain = apps.get_model('desecapi', 'Domain') - domains = list(Domain.objects.all()) - for domain in domains: - domain.name = domain.name.lower() - Domain.objects.bulk_update(domains, ['name'], batch_size=500) - - # RRsets - RRset = apps.get_model('desecapi', 'RRset') - rrsets = list(RRset.objects.all()) - for rrset in rrsets: - rrset.subname = rrset.subname.lower() - RRset.objects.bulk_update(rrsets, ['subname'], batch_size=500) - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0001_initial_squashed'), - ] - - operations = [ - migrations.AlterField( - model_name='domain', - name='name', - field=models.CharField(max_length=191, unique=True, validators=[desecapi.models.validate_lower, django.core.validators.RegexValidator(code='invalid_domain_name', message='Domain name malformed.', regex='^[a-z0-9_.-]+$')]), - ), - migrations.AlterField( - model_name='domain', - name='published', - field=models.DateTimeField(blank=True, null=True), - ), - migrations.AlterField( - model_name='rrset', - name='subname', - field=models.CharField(blank=True, max_length=178, validators=[desecapi.models.validate_lower, django.core.validators.RegexValidator(code='invalid_subname', message='Subname malformed.', regex='^[*]?[a-z0-9_.-]*$')]), - ), - migrations.AlterField( - model_name='rrset', - name='type', - field=models.CharField(max_length=10, validators=[desecapi.models.validate_upper, django.core.validators.RegexValidator(code='invalid_type', message='Type malformed.', regex='^[A-Z][A-Z0-9]*$')]), - ), - migrations.RunPython(lowercase_names, reverse_code=migrations.RunPython.noop), - ] diff --git a/api/desecapi/migrations/0003_validation.py b/api/desecapi/migrations/0003_validation.py deleted file mode 100644 index 517478158..000000000 --- a/api/desecapi/migrations/0003_validation.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 2.2.2 on 2019-06-28 18:16 - -import desecapi.models -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0002_lowercase_domains_and_subnames'), - ] - - operations = [ - migrations.AlterField( - model_name='domain', - name='name', - field=models.CharField(max_length=191, unique=True, validators=[desecapi.models.validate_lower, django.core.validators.RegexValidator(code='invalid_domain_name', message='Invalid value (not a DNS name).', regex='^[a-z0-9_.-]*[a-z]$')]), - ), - migrations.AlterField( - model_name='rrset', - name='subname', - field=models.CharField(blank=True, max_length=178, validators=[desecapi.models.validate_lower, django.core.validators.RegexValidator(code='invalid_subname', message="Subname can only use (lowercase) a-z, 0-9, ., -, and _, may start with a '*.', or just be '*'.", regex='^([*]|(([*][.])?[a-z0-9_.-]*))$')]), - ), - migrations.AlterField( - model_name='rrset', - name='type', - field=models.CharField(max_length=10, validators=[desecapi.models.validate_upper, django.core.validators.RegexValidator(code='invalid_type', message='Type must be uppercase alphanumeric and start with a letter.', regex='^[A-Z][A-Z0-9]*$')]), - ), - ] diff --git a/api/desecapi/migrations/0004_domain_minimum_ttl.py b/api/desecapi/migrations/0004_domain_minimum_ttl.py deleted file mode 100644 index 3829223a2..000000000 --- a/api/desecapi/migrations/0004_domain_minimum_ttl.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 2.2.3 on 2019-07-19 12:14 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0003_validation'), - ] - - operations = [ - migrations.AddField( - model_name='domain', - name='minimum_ttl', - field=models.PositiveIntegerField(default=60), - ), - migrations.AlterField( - model_name='rrset', - name='ttl', - field=models.PositiveIntegerField(), - ), - ] diff --git a/api/desecapi/migrations/0005_user_model_cleanup.py b/api/desecapi/migrations/0005_user_model_cleanup.py deleted file mode 100644 index 4cf082c35..000000000 --- a/api/desecapi/migrations/0005_user_model_cleanup.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 2.2.2 on 2019-06-28 18:23 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0004_domain_minimum_ttl'), - ] - - operations = [ - migrations.RemoveField( - model_name='user', - name='dyn', - ), - migrations.RemoveField( - model_name='user', - name='locked', - ), - ] diff --git a/api/desecapi/migrations/0006_authenticated_actions.py b/api/desecapi/migrations/0006_authenticated_actions.py deleted file mode 100644 index 3534e23ff..000000000 --- a/api/desecapi/migrations/0006_authenticated_actions.py +++ /dev/null @@ -1,79 +0,0 @@ -# Generated by Django 2.2.1 on 2019-09-21 11:39 - -import datetime -from django.db import migrations, models -import django.db.models.deletion -from django.utils import timezone - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0005_user_model_cleanup'), - ] - - operations = [ - migrations.CreateModel( - name='AuthenticatedAction', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created', models.PositiveIntegerField(default=lambda: int(datetime.timestamp(timezone.now())))), - ], - options={ - 'managed': False, - }, - ), - migrations.CreateModel( - name='AuthenticatedUserAction', - fields=[ - ('authenticatedaction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.AuthenticatedAction')), - ], - options={ - 'managed': False, - }, - bases=('desecapi.authenticatedaction',), - ), - migrations.CreateModel( - name='AuthenticatedActivateUserAction', - fields=[ - ('authenticateduseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.AuthenticatedUserAction')), - ('domain', models.CharField(max_length=191)), - ], - options={ - 'managed': False, - }, - bases=('desecapi.authenticateduseraction',), - ), - migrations.CreateModel( - name='AuthenticatedChangeEmailUserAction', - fields=[ - ('authenticateduseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.AuthenticatedUserAction')), - ('new_email', models.EmailField(max_length=254)), - ], - options={ - 'managed': False, - }, - bases=('desecapi.authenticateduseraction',), - ), - migrations.CreateModel( - name='AuthenticatedDeleteUserAction', - fields=[ - ('authenticateduseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.AuthenticatedUserAction')), - ], - options={ - 'managed': False, - }, - bases=('desecapi.authenticateduseraction',), - ), - migrations.CreateModel( - name='AuthenticatedResetPasswordUserAction', - fields=[ - ('authenticateduseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.AuthenticatedUserAction')), - ('new_password', models.CharField(max_length=128)), - ], - options={ - 'managed': False, - }, - bases=('desecapi.authenticateduseraction',), - ), - ] diff --git a/api/desecapi/migrations/0007_remove_user_registration_remote_ip.py b/api/desecapi/migrations/0007_remove_user_registration_remote_ip.py deleted file mode 100644 index 55e3ac6be..000000000 --- a/api/desecapi/migrations/0007_remove_user_registration_remote_ip.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 2.2.1 on 2019-09-22 10:19 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0006_authenticated_actions'), - ] - - operations = [ - migrations.RemoveField( - model_name='user', - name='registration_remote_ip', - ), - ] diff --git a/api/desecapi/migrations/0008_captcha.py b/api/desecapi/migrations/0008_captcha.py deleted file mode 100644 index c7ac27c4c..000000000 --- a/api/desecapi/migrations/0008_captcha.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 2.2.1 on 2019-09-22 10:26 - -import desecapi.models -from django.db import migrations, models -import django.db.models.deletion -import uuid - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0007_remove_user_registration_remote_ip'), - ] - - operations = [ - migrations.CreateModel( - name='Captcha', - fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('created', models.DateTimeField(auto_now_add=True)), - ('content', models.CharField(default=desecapi.models.captcha_default_content, max_length=24)), - ], - ), - ] diff --git a/api/desecapi/migrations/0009_domain_minimum_ttl_default.py b/api/desecapi/migrations/0009_domain_minimum_ttl_default.py deleted file mode 100644 index 3565ef69d..000000000 --- a/api/desecapi/migrations/0009_domain_minimum_ttl_default.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 2.2.6 on 2019-10-10 15:07 - -import desecapi.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0008_captcha'), - ] - - operations = [ - migrations.AlterField( - model_name='domain', - name='minimum_ttl', - field=models.PositiveIntegerField(default=desecapi.models.get_minimum_ttl_default), - ), - ] - diff --git a/api/desecapi/migrations/0010_hash_tokens_and_switch_to_uuid.py b/api/desecapi/migrations/0010_hash_tokens_and_switch_to_uuid.py deleted file mode 100644 index 66be986f0..000000000 --- a/api/desecapi/migrations/0010_hash_tokens_and_switch_to_uuid.py +++ /dev/null @@ -1,52 +0,0 @@ -# Generated by Django 2.2.7 on 2019-11-11 20:14 - -from django.contrib.auth.hashers import make_password -from django.db import migrations, models, transaction -import uuid - - -def migrate_data(apps, schema_editor): - Token = apps.get_model('desecapi', 'Token') - tokens = Token.objects.exclude(key__contains='$').all() - with transaction.atomic(): - for token in tokens: - hashed = make_password(token.key, salt='static', hasher='pbkdf2_sha256_iter1') - Token.objects.filter(key=token.key).update(id=uuid.uuid4().hex, key=hashed) - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0009_domain_minimum_ttl_default'), - ] - - operations = [ - migrations.AlterModelOptions( - name='token', - options={'verbose_name': 'Token', 'verbose_name_plural': 'Tokens'}, - ), - migrations.AlterUniqueTogether( - name='token', - unique_together=set(), - ), - migrations.RemoveField( - model_name='token', - name='user_specific_id', - ), - migrations.AlterField( - model_name='token', - name='key', - field=models.CharField(db_index=True, max_length=128, unique=True, verbose_name='Key'), - ), - migrations.AlterField( - model_name='token', - name='id', - field=models.CharField(default=uuid.uuid4, max_length=32, primary_key=True, serialize=False), - ), - migrations.RunPython(migrate_data, reverse_code=migrations.RunPython.noop), - migrations.AlterField( - model_name='token', - name='id', - field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), - ), - ] diff --git a/api/desecapi/migrations/0011_user_id_to_uuid.py b/api/desecapi/migrations/0011_user_id_to_uuid.py deleted file mode 100644 index b729db52f..000000000 --- a/api/desecapi/migrations/0011_user_id_to_uuid.py +++ /dev/null @@ -1,88 +0,0 @@ -from django.conf import settings -from django.db import migrations, models, transaction -import django.db.models.deletion -import uuid - - -def migrate_data(apps, schema_editor): - # SQLite does not support altering constraints. However, we use it for tests only, and there's no data to migrate. - if schema_editor.connection.vendor == 'sqlite': - return - - def _sql_add_cascading_user_constraint(model_name, field_name): - return f'ALTER TABLE desecapi_{model_name}' \ - f' ADD CONSTRAINT desecapi_{model_name}_{field_name}_id_update_cascade' \ - f' FOREIGN KEY (`{field_name}_id`) REFERENCES `desecapi_user` (`id`) ON UPDATE CASCADE' - - def _sql_drop_cascading_user_constraint(model_name, field_name): - return f'ALTER TABLE desecapi_{model_name} DROP CONSTRAINT desecapi_{model_name}_{field_name}_id_update_cascade' - - # Add cascading foreign key constraints. - # This has to be done after removing the regular constraints using migrations.AlterField. If done the other - # way around, AlterField will drop the cascading constraint. - schema_editor.execute(_sql_add_cascading_user_constraint('domain', 'owner')), - schema_editor.execute(_sql_add_cascading_user_constraint('token', 'user')), - - # Repopulate user ID fields - User = apps.get_model('desecapi', 'User') - with transaction.atomic(): - for user in User.objects.all(): - User.objects.filter(email=user.email).update(id=uuid.uuid4().hex) - - # Remove cascading foreign key constraints - schema_editor.execute(_sql_drop_cascading_user_constraint('domain', 'owner')), - schema_editor.execute(_sql_drop_cascading_user_constraint('token', 'user')), - - -class Migration(migrations.Migration): - dependencies = [ - ('desecapi', '0010_hash_tokens_and_switch_to_uuid'), - ] - - operations = [ - # Switch to intermediate field type - migrations.AlterField( - model_name='user', - name='id', - field=models.CharField(default=uuid.uuid4, max_length=32, primary_key=True, serialize=False), - ), - - # Remove regular foreign key constraints. - # This is the migration Django generates when you set db_constraint=False on the model field. - migrations.AlterField( - model_name='domain', - name='owner', - field=models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.PROTECT, - related_name='domains', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='token', - name='user', - field=models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, - related_name='auth_tokens', to=settings.AUTH_USER_MODEL, verbose_name='User'), - ), - - # Repopulate user IDs with random UUIDs - migrations.RunPython(migrate_data, migrations.RunPython.noop, atomic=False), - - # Restore regular foreign key constraints - migrations.AlterField( - model_name='token', - name='user', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='auth_tokens', - to=settings.AUTH_USER_MODEL, verbose_name='User'), - ), - migrations.AlterField( - model_name='domain', - name='owner', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='domains', - to=settings.AUTH_USER_MODEL), - ), - - # Switch to final field type - migrations.AlterField( - model_name='user', - name='id', - field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), - ), - ] diff --git a/api/desecapi/migrations/0012_volatile_donations.py b/api/desecapi/migrations/0012_volatile_donations.py deleted file mode 100644 index 7f0096adf..000000000 --- a/api/desecapi/migrations/0012_volatile_donations.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 2.2.7 on 2019-11-27 15:30 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0011_user_id_to_uuid'), - ] - - operations = [ - migrations.AlterModelOptions( - name='donation', - options={'managed': False}, - ), - ] diff --git a/api/desecapi/migrations/0013_token_last_used.py b/api/desecapi/migrations/0013_token_last_used.py deleted file mode 100644 index 683e9d472..000000000 --- a/api/desecapi/migrations/0013_token_last_used.py +++ /dev/null @@ -1,45 +0,0 @@ -# Generated by Django 3.0.5 on 2020-04-29 17:41 - -import desecapi.models -import django.core.validators -from django.db import migrations, models, transaction - - -def migrate_data(apps, schema_editor): - Token = apps.get_model('desecapi', 'Token') - tokens = Token.objects.filter(last_used__isnull=True).all() - with transaction.atomic(): - for token in tokens: - # Don't suggest that existing tokens have not been in use. - Token.objects.filter(pk=token.id).update(last_used=token.user.last_login) - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0012_volatile_donations'), - ] - - operations = [ - migrations.AddField( - model_name='token', - name='last_used', - field=models.DateTimeField(blank=True, null=True), - ), - migrations.AlterField( - model_name='token', - name='name', - field=models.CharField(blank=True, max_length=64, verbose_name='Name'), - ), - migrations.AlterField( - model_name='domain', - name='name', - field=models.CharField(max_length=191, unique=True, validators=[desecapi.models.validate_lower, django.core.validators.RegexValidator(code='invalid_domain_name', message='Invalid value (not a DNS name).', regex='^([a-z0-9_-]{1,63}\\.)*[a-z]{1,63}$')]), - ), - migrations.AlterField( - model_name='rr', - name='content', - field=models.CharField(max_length=500), - ), - migrations.RunPython(migrate_data, migrations.RunPython.noop), - ] diff --git a/api/desecapi/migrations/0014_rrset_touched.py b/api/desecapi/migrations/0014_rrset_touched.py deleted file mode 100644 index 657cedcf6..000000000 --- a/api/desecapi/migrations/0014_rrset_touched.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 3.0.6 on 2020-05-21 10:59 - -import desecapi.models -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0013_token_last_used'), - ] - - operations = [ - migrations.RenameField( - model_name='rrset', - old_name='updated', - new_name='touched', - ), - ] diff --git a/api/desecapi/migrations/0015_rrset_touched_auto_now.py b/api/desecapi/migrations/0015_rrset_touched_auto_now.py deleted file mode 100644 index 5e59604bb..000000000 --- a/api/desecapi/migrations/0015_rrset_touched_auto_now.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.0.6 on 2020-05-22 12:10 - -from django.db import migrations, models - - -def migrate_data(apps, schema_editor): - RRset = apps.get_model('desecapi', 'RRset') - RRset.objects.filter(touched__isnull=True).update(touched=models.F('created')) - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0014_rrset_touched'), - ] - - operations = [ - migrations.RunPython(migrate_data, migrations.RunPython.noop), - migrations.AlterField( - model_name='rrset', - name='touched', - field=models.DateTimeField(auto_now=True), - ), - ] diff --git a/api/desecapi/migrations/0016_domain_renewal.py b/api/desecapi/migrations/0016_domain_renewal.py deleted file mode 100644 index 1e9257ec2..000000000 --- a/api/desecapi/migrations/0016_domain_renewal.py +++ /dev/null @@ -1,62 +0,0 @@ -# Generated by Django 3.0.6 on 2020-06-29 19:15 - -import desecapi.models -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion -import re - - -class Migration(migrations.Migration): - - dependencies = [ - ('desecapi', '0015_rrset_touched_auto_now'), - ] - - operations = [ - migrations.CreateModel( - name='AuthenticatedBasicUserAction', - fields=[ - ('authenticatedaction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.AuthenticatedAction')), - ], - options={ - 'managed': False, - }, - bases=('desecapi.authenticatedaction',), - ), - migrations.AddField( - model_name='domain', - name='renewal_changed', - field=models.DateTimeField(auto_now_add=True), - ), - migrations.AddField( - model_name='domain', - name='renewal_state', - field=models.IntegerField(choices=[(1, 'Fresh'), (2, 'Notified'), (3, 'Warned')], default=1), - ), - migrations.AlterField( - model_name='domain', - name='name', - field=models.CharField(max_length=191, unique=True, validators=[desecapi.models.validate_lower, django.core.validators.RegexValidator(code='invalid_domain_name', flags=re.RegexFlag['IGNORECASE'], message='Domain names must be labels separated by dots. Labels may consist of up to 63 letters, digits, hyphens, and underscores. The last label may only contain letters.', regex='^(([a-z0-9_-]{1,63})\\.)*[a-z]{1,63}$')]), - ), - migrations.CreateModel( - name='AuthenticatedDomainBasicUserAction', - fields=[ - ('authenticatedbasicuseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.AuthenticatedBasicUserAction')), - ], - options={ - 'managed': False, - }, - bases=('desecapi.authenticatedbasicuseraction',), - ), - migrations.CreateModel( - name='AuthenticatedRenewDomainBasicUserAction', - fields=[ - ('authenticateddomainbasicuseraction_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='desecapi.AuthenticatedDomainBasicUserAction')), - ], - options={ - 'managed': False, - }, - bases=('desecapi.authenticateddomainbasicuseraction',), - ), - ] From 149150c93c7261e9d5d87f8f7f3bae14fa53fd60 Mon Sep 17 00:00:00 2001 From: Nils Wisiol Date: Wed, 26 Aug 2020 09:44:29 +0200 Subject: [PATCH 3/4] fix(api): adds no-op migration that Django thinks is missing --- .../migrations/0002_unmanaged_donations.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 api/desecapi/migrations/0002_unmanaged_donations.py diff --git a/api/desecapi/migrations/0002_unmanaged_donations.py b/api/desecapi/migrations/0002_unmanaged_donations.py new file mode 100644 index 000000000..1b7f1cf5e --- /dev/null +++ b/api/desecapi/migrations/0002_unmanaged_donations.py @@ -0,0 +1,20 @@ +# Generated by Django 3.1 on 2020-08-26 07:42 + +# It appears that this migration is "unsquashable". If you squash it in 0001, Django will think it's missing and +# generate it next time again. + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('desecapi', '0001_initial_squashed_again'), + ] + + operations = [ + migrations.AlterModelOptions( + name='donation', + options={'managed': False}, + ), + ] From 50b7baeda75d9e9deb388e4666c372b32d959e95 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Wed, 6 Nov 2019 21:08:36 +0100 Subject: [PATCH 4/4] BREAKING feat(dbapi): switch to postgres, fixes #430 Migration instructions: https://github.com/desec-io/desec-stack/pull/432#issuecomment-680785746 --- README.md | 12 ++++++------ api/Dockerfile | 4 ++-- api/api/settings.py | 11 +---------- api/desecapi/serializers.py | 9 ++------- api/requirements.txt | 2 +- api/wait | 2 +- dbapi/Dockerfile | 17 +++++++++-------- .../docker-entrypoint-initdb.d/init-user-db.sh | 15 +++++++++++++++ dbapi/entrypoint-wrapper.sh | 6 ++++++ dbapi/initdb.d/00-init.sh | 6 ------ dbapi/initdb.d/00-init.sql | 2 -- dbapi/initdb.d/00-init.sql.var | 7 ------- dbapi/initdb.d/99-finish.sql | 4 ---- dbapi/pg_hba.conf | 16 ++++++++++++++++ docker-compose.yml | 15 +++++++++++---- 15 files changed, 70 insertions(+), 58 deletions(-) create mode 100644 dbapi/docker-entrypoint-initdb.d/init-user-db.sh create mode 100755 dbapi/entrypoint-wrapper.sh delete mode 100644 dbapi/initdb.d/00-init.sh delete mode 100644 dbapi/initdb.d/00-init.sql delete mode 100644 dbapi/initdb.d/00-init.sql.var delete mode 100644 dbapi/initdb.d/99-finish.sql create mode 100644 dbapi/pg_hba.conf diff --git a/README.md b/README.md index a07cd57c0..945c10492 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This is a docker-compose application providing the basic stack for deSEC name se - `nslord`: Eventually authoritative DNS server (PowerDNS). DNSSEC keying material is generated here. - `nsmaster`: Stealth authoritative DNS server (PowerDNS). Receives fully signed AXFR zone transfers from `nslord`. No access to keys. - `api`: RESTful API to create deSEC users and domains, see [documentation](https://desec.readthedocs.io/). -- `dbapi`, `dblord`, `dbmaster`: MariaDB database services for `api`, `nslord`, and `nsmaster`, respectively. +- `dbapi`, `dblord`, `dbmaster`: Postgres database for `api`, MariaDB databases for `nslord` and `nsmaster`, respectively. - `www`: nginx instance serving static web site content and proxying to `api` - `celery`: A shadow instance of the `api` code for performing asynchronous tasks (email delivery). - `rabbitmq`: `celery`'s queue @@ -52,7 +52,7 @@ Although most configuration is contained in this repository, some external depen - `DESECSTACK_API_EMAIL_PORT`: port for sending email - `DESECSTACK_API_SECRETKEY`: Django secret - `DESECSTACK_API_PSL_RESOLVER`: Resolver IP address to use for PSL lookups. If empty, the system's default resolver is used. - - `DESECSTACK_DBAPI_PASSWORD_desec`: mysql password for desecapi + - `DESECSTACK_DBAPI_PASSWORD_desec`: database password for desecapi - `DESECSTACK_MINIMUM_TTL_DEFAULT`: minimum TTL users can set for RRsets. The setting is per domain, and the default defined here is used on domain creation. - nslord-related - `DESECSTACK_DBLORD_PASSWORD_pdns`: mysql password for pdns on nslord @@ -82,8 +82,8 @@ Production: Storage ------- -All important data is stored in the databases managed by the `db*` containers. They use Docker volumes which, by default, reside in `/var/lib/docker/volumes/desecstack_{dbapi,dblord,dbmaster}_mysql`. -This is the location you will want to back up. (Be sure to follow standard MySQL backup practices, i.e. make sure things are consistent.) +All important data is stored in the databases managed by the `db*` containers. They use Docker volumes which, by default, reside in `/var/lib/docker/volumes/desec-stack_{dbapi_postgres,dblord_mysql,dbmaster_mysql}`. +This is the location you will want to back up. (Be sure to follow standard MySQL/Postgres backup practices, i.e. make sure things are consistent.) API Versions and Roadmap ------------------------ @@ -135,10 +135,10 @@ While there are certainly many ways to get started hacking desec-stack, here is For desec-stack, [docker](https://docs.docker.com/install/linux/docker-ce/ubuntu/) and [docker-compose](https://docs.docker.com/compose/install/) are required. Further tools that are required to start hacking are git and curl. Recommended, but not strictly required for desec-stack development is to use certbot along with Let's Encrypt and PyCharm. - jq, httpie, libmariadbclient-dev, python3-dev (>= 3.8) and python3-venv (>= 3.8) are useful if you want to follow this guide. + jq, httpie, libmariadbclient-dev, libpq-dev, python3-dev (>= 3.8) and python3-venv (>= 3.8) are useful if you want to follow this guide. The webapp requires nodejs. To install everything you need for this guide except docker and docker-compose, use - sudo apt install certbot curl git httpie jq libmariadbclient-dev nodejs npm python3-dev python3-venv libmemcached-dev + sudo apt install certbot curl git httpie jq libmariadbclient-dev libpq-dev nodejs npm python3-dev python3-venv libmemcached-dev 1. **Get the code.** Clone this repository to your favorite location. diff --git a/api/Dockerfile b/api/Dockerfile index f5a752de7..6c7443296 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.8-alpine -RUN apk add --no-cache bash dcron sqlite +RUN apk add --no-cache bash dcron postgresql-client sqlite RUN mkdir /usr/src/app WORKDIR /usr/src/app @@ -9,7 +9,7 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 ENV PIP_NO_CACHE_DIR=1 COPY requirements.txt /usr/src/app/ -RUN apk add --no-cache gcc freetype-dev libffi-dev musl-dev libmemcached-dev mariadb-connector-c-dev jpeg-dev zlib-dev \ +RUN apk add --no-cache gcc freetype-dev libffi-dev musl-dev libmemcached-dev postgresql-dev jpeg-dev zlib-dev \ && pip install -r requirements.txt \ && apk --no-cache del gcc RUN pip freeze diff --git a/api/api/settings.py b/api/api/settings.py index 2321b8494..7b68ffa7f 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -62,21 +62,12 @@ DATABASES = { 'default': { - 'ENGINE': 'django_prometheus.db.backends.mysql', + 'ENGINE': 'django_prometheus.db.backends.postgresql', 'NAME': 'desec', 'USER': 'desec', 'PASSWORD': os.environ['DESECSTACK_DBAPI_PASSWORD_desec'], 'HOST': 'dbapi', - 'OPTIONS': { - 'charset': 'utf8mb4', - 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'", - }, - 'TEST': { - 'CHARSET': 'utf8mb4', - 'COLLATION': 'utf8mb4_bin', - }, }, - } CACHES = { diff --git a/api/desecapi/serializers.py b/api/desecapi/serializers.py index 2e40ae95c..a1776323e 100644 --- a/api/desecapi/serializers.py +++ b/api/desecapi/serializers.py @@ -492,14 +492,9 @@ def is_empty(data_item): # time of check (does it exist?) and time of action (create vs update) are different, # so for parallel requests, we can get integrity errors due to duplicate keys. - # This will be considered a 429-error, even though re-sending the request will be successful. + # We knew how to handle this with MySQL, but after switching for Postgres, we don't. + # Re-raise it so we get an email based on which we can learn and improve error handling. except OperationalError as e: - try: - if e.args[0] == 1213: - # 1213 is mysql for deadlock, other OperationalErrors are treated elsewhere or not treated at all - raise ConcurrencyException from e - except (AttributeError, KeyError): - pass raise e except (IntegrityError, models.RRset.DoesNotExist) as e: raise ConcurrencyException from e diff --git a/api/requirements.txt b/api/requirements.txt index 07ee9971d..fdd5de513 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -9,7 +9,7 @@ django-celery-email~=3.0.0 django-prometheus~=2.0.0 dnspython~=1.16.0 httpretty~=0.9.0 -mysqlclient~=1.4.0 +psycopg2~=2.8.5 prometheus-client~=0.8.0 # added to control django-prometheus' dependency version psl-dns~=1.0 pylibmc~=1.6.1 diff --git a/api/wait b/api/wait index 7dc838360..4234586bb 100755 --- a/api/wait +++ b/api/wait @@ -2,7 +2,7 @@ set -e # wait for api database to come up -host=dbapi; port=3306; n=120; i=0; while ! (echo > /dev/tcp/$host/$port) 2> /dev/null; do [[ $i -eq $n ]] && >&2 echo "$host:$port not up after $n seconds, exiting" && exit 1; echo "waiting for $host:$port to come up"; sleep 1; i=$((i+1)); done +host=dbapi; port=5432; n=120; i=0; while ! (echo > /dev/tcp/$host/$port) 2> /dev/null; do [[ $i -eq $n ]] && >&2 echo "$host:$port not up after $n seconds, exiting" && exit 1; echo "waiting for $host:$port to come up"; sleep 1; i=$((i+1)); done # wait for pdns api to come up host=nslord; port=8081; n=120; i=0; while ! (echo > /dev/tcp/$host/$port) 2> /dev/null; do [[ $i -eq $n ]] && >&2 echo "$host:$port not up after $n seconds, exiting" && exit 1; echo "waiting for $host:$port to come up"; sleep 1; i=$((i+1)); done diff --git a/dbapi/Dockerfile b/dbapi/Dockerfile index 401b83209..82f75d135 100644 --- a/dbapi/Dockerfile +++ b/dbapi/Dockerfile @@ -1,13 +1,14 @@ -FROM mariadb:10.3 +FROM postgres:12-alpine -# Use random throw-away root password. Our init scripts switch authentication to socket logins only -ENV MYSQL_RANDOM_ROOT_PASSWORD=yes +RUN apk add --no-cache pwgen -# install tools used in init script -RUN set -ex && apt-get update && apt-get -y install gettext-base && apt-get clean && rm -rf /var/lib/apt/lists/* +ADD docker-entrypoint-initdb.d /docker-entrypoint-initdb.d -COPY initdb.d/* /docker-entrypoint-initdb.d/ -RUN chown -R mysql:mysql /docker-entrypoint-initdb.d/ +USER postgres # mountable storage -VOLUME /var/lib/mysql +VOLUME /var/lib/postgresql/data + +COPY entrypoint-wrapper.sh /usr/local/bin/ +ENTRYPOINT ["entrypoint-wrapper.sh"] +CMD ["postgres"] diff --git a/dbapi/docker-entrypoint-initdb.d/init-user-db.sh b/dbapi/docker-entrypoint-initdb.d/init-user-db.sh new file mode 100644 index 000000000..70996be33 --- /dev/null +++ b/dbapi/docker-entrypoint-initdb.d/init-user-db.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +# Get the postgres user or set it to a default value +if [ -n $POSTGRES_USER ]; then pg_user=$POSTGRES_USER; else pg_user="postgres"; fi +# Get the postgres db or set it to a default value +if [ -n $POSTGRES_DB ]; then pg_db=$POSTGRES_DB; else pg_db=$POSTGRES_USER; fi + +if [ -n "$POSTGRES_NON_ROOT_USER" ]; then +psql -v ON_ERROR_STOP=1 --username "$pg_user" --dbname "$pg_db" <<-EOSQL + CREATE USER $POSTGRES_NON_ROOT_USER with encrypted password '$POSTGRES_NON_ROOT_USER_PASSWORD'; + GRANT CREATE, CONNECT ON DATABASE $pg_db TO $POSTGRES_NON_ROOT_USER; + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, UPDATE, INSERT, DELETE, REFERENCES ON TABLES TO $POSTGRES_NON_ROOT_USER; +EOSQL +fi diff --git a/dbapi/entrypoint-wrapper.sh b/dbapi/entrypoint-wrapper.sh new file mode 100755 index 000000000..0bc605fc5 --- /dev/null +++ b/dbapi/entrypoint-wrapper.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -Eeo pipefail + +# This password is set for the postgres user when initializing the database. It is not needed and thus not printed. +export POSTGRES_PASSWORD=$(pwgen -1 -s 32) +/usr/local/bin/docker-entrypoint.sh "$@" diff --git a/dbapi/initdb.d/00-init.sh b/dbapi/initdb.d/00-init.sh deleted file mode 100644 index 71097be77..000000000 --- a/dbapi/initdb.d/00-init.sh +++ /dev/null @@ -1,6 +0,0 @@ -# https://stackoverflow.com/questions/59895/can-a-bash-script-tell-which-directory-it-is-stored-in -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -for file in $DIR/*.sql.var; do - envsubst < $file > $DIR/`basename $file .var` -done diff --git a/dbapi/initdb.d/00-init.sql b/dbapi/initdb.d/00-init.sql deleted file mode 100644 index 83756fd26..000000000 --- a/dbapi/initdb.d/00-init.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file is required to exist and will be overriden by 00-init.sh. --- If it is created only by 00-init.sh, the entrypoint script will miss it. diff --git a/dbapi/initdb.d/00-init.sql.var b/dbapi/initdb.d/00-init.sql.var deleted file mode 100644 index 6c61b6c19..000000000 --- a/dbapi/initdb.d/00-init.sql.var +++ /dev/null @@ -1,7 +0,0 @@ --- deSEC user and domain database -CREATE DATABASE desec CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; -CREATE USER 'desec'@'${DESECSTACK_IPV4_REAR_PREFIX16}.5.%' IDENTIFIED BY '${DESECSTACK_DBAPI_PASSWORD_desec}'; -GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES, INDEX, CREATE, ALTER, DROP ON desec.* TO 'desec'@'${DESECSTACK_IPV4_REAR_PREFIX16}.5.%'; - --- privileges for deSEC test database -GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES, INDEX, CREATE, ALTER, DROP ON test_desec.* TO 'desec'@'${DESECSTACK_IPV4_REAR_PREFIX16}.5.%'; diff --git a/dbapi/initdb.d/99-finish.sql b/dbapi/initdb.d/99-finish.sql deleted file mode 100644 index 30349eb1c..000000000 --- a/dbapi/initdb.d/99-finish.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Narrow down root logins -DROP USER 'root'@'%'; -INSTALL PLUGIN unix_socket SONAME 'auth_socket'; -ALTER USER 'root'@'localhost' IDENTIFIED VIA unix_socket; diff --git a/dbapi/pg_hba.conf b/dbapi/pg_hba.conf new file mode 100644 index 000000000..eb3bb210a --- /dev/null +++ b/dbapi/pg_hba.conf @@ -0,0 +1,16 @@ +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all trust +# IPv4 local connections: +#host all all 127.0.0.1/32 scram-sha-256 +# IPv6 local connections: +#host all all ::1/128 scram-sha-256 +# Allow replication connections from localhost, by a user with the +# replication privilege. +#local replication all trust +#host replication all 127.0.0.1/32 scram-sha-256 +#host replication all ::1/128 scram-sha-256 + +host desec desec all scram-sha-256 +host all all all reject diff --git a/docker-compose.yml b/docker-compose.yml index e9697fbf7..0184f0bde 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,14 +56,21 @@ services: build: dbapi image: desec/dedyn-dbapi:latest init: true - user: mysql:mysql + user: postgres:postgres + shm_size: 256M volumes: - - dbapi_mysql:/var/lib/mysql + - dbapi_postgres:/var/lib/postgresql/data + - ./dbapi/pg_hba.conf:/usr/local/src/pg_hba.conf:ro environment: - DESECSTACK_IPV4_REAR_PREFIX16 - - DESECSTACK_DBAPI_PASSWORD_desec + - POSTGRES_DB=desec + - POSTGRES_HOST_AUTH_METHOD=reject + - POSTGRES_INITDB_ARGS=--auth-host=scram-sha-256 + - POSTGRES_NON_ROOT_USER=desec + - POSTGRES_NON_ROOT_USER_PASSWORD=${DESECSTACK_DBAPI_PASSWORD_desec} networks: - rearapi_dbapi + command: ["postgres", "-c", "hba_file=/usr/local/src/pg_hba.conf"] logging: driver: "syslog" options: @@ -352,7 +359,7 @@ services: restart: unless-stopped volumes: - dbapi_mysql: + dbapi_postgres: dblord_mysql: dbmaster_mysql: openvpn-server_logs: