From 9f8f64b9ec53090ca7d8921f9ea7fbcf2f4f7bce Mon Sep 17 00:00:00 2001 From: kevgliss Date: Fri, 29 Jan 2016 09:22:12 -0800 Subject: [PATCH] removing pre 2.0 migration scripts, and adding documentation for correct path during init --- docs/quickstart/index.rst | 1 + lemur/migrations/versions/1ff763f5b80b_.py | 42 ---- lemur/migrations/versions/33de094da890_.py | 2 +- lemur/migrations/versions/3b718f59b8ce_.py | 49 ---- lemur/migrations/versions/4bcfa2c36623_.py | 26 --- lemur/migrations/versions/4c8915e461b3_.py | 41 ---- lemur/migrations/versions/4dc5ddd111b8_.py | 31 --- lemur/migrations/versions/ed422fc58ba_.py | 255 --------------------- 8 files changed, 2 insertions(+), 445 deletions(-) delete mode 100644 lemur/migrations/versions/1ff763f5b80b_.py delete mode 100644 lemur/migrations/versions/3b718f59b8ce_.py delete mode 100644 lemur/migrations/versions/4bcfa2c36623_.py delete mode 100644 lemur/migrations/versions/4c8915e461b3_.py delete mode 100644 lemur/migrations/versions/4dc5ddd111b8_.py delete mode 100644 lemur/migrations/versions/ed422fc58ba_.py diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index e6d4186fe3..4e0f2d7cd1 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -158,6 +158,7 @@ Additional notifications can be created through the UI or API. See :ref:`Creati .. code-block:: bash + $ cd /www/lemur/lemur $ lemur init .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. diff --git a/lemur/migrations/versions/1ff763f5b80b_.py b/lemur/migrations/versions/1ff763f5b80b_.py deleted file mode 100644 index e4a9af228a..0000000000 --- a/lemur/migrations/versions/1ff763f5b80b_.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Adding in models for certificate sources - -Revision ID: 1ff763f5b80b -Revises: 4dc5ddd111b8 -Create Date: 2015-08-01 15:24:20.412725 - -""" - -# revision identifiers, used by Alembic. -revision = '1ff763f5b80b' -down_revision = '4dc5ddd111b8' - -from alembic import op -import sqlalchemy as sa - -import sqlalchemy_utils - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.create_table('sources', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('label', sa.String(length=32), nullable=True), - sa.Column('options', sqlalchemy_utils.types.json.JSONType(), nullable=True), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('plugin_name', sa.String(length=32), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('certificate_source_associations', - sa.Column('source_id', sa.Integer(), nullable=True), - sa.Column('certificate_id', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'), - sa.ForeignKeyConstraint(['source_id'], ['destinations.id'], ondelete='cascade') - ) - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.drop_table('certificate_source_associations') - op.drop_table('sources') - ### end Alembic commands ### diff --git a/lemur/migrations/versions/33de094da890_.py b/lemur/migrations/versions/33de094da890_.py index a4ca3810e0..76624e9667 100644 --- a/lemur/migrations/versions/33de094da890_.py +++ b/lemur/migrations/versions/33de094da890_.py @@ -8,7 +8,7 @@ # revision identifiers, used by Alembic. revision = '33de094da890' -down_revision = 'ed422fc58ba' +down_revision = None from alembic import op import sqlalchemy as sa diff --git a/lemur/migrations/versions/3b718f59b8ce_.py b/lemur/migrations/versions/3b718f59b8ce_.py deleted file mode 100644 index 1902e9d76e..0000000000 --- a/lemur/migrations/versions/3b718f59b8ce_.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Refactors Accounts to Destinations - -Revision ID: 3b718f59b8ce -Revises: None -Create Date: 2015-07-09 17:44:55.626221 - -""" - -# revision identifiers, used by Alembic. -revision = '3b718f59b8ce' -down_revision = None - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.drop_table('certificate_account_associations') - op.drop_table('accounts') - op.add_column('destinations', sa.Column('plugin_name', sa.String(length=32), nullable=True)) - op.drop_index('ix_elbs_account_id', table_name='elbs') - op.drop_constraint(u'elbs_account_id_fkey', 'elbs', type_='foreignkey') - op.drop_column('elbs', 'account_id') - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.add_column('elbs', sa.Column('account_id', sa.BIGINT(), autoincrement=False, nullable=True)) - op.create_foreign_key(u'elbs_account_id_fkey', 'elbs', 'accounts', ['account_id'], ['id']) - op.create_index('ix_elbs_account_id', 'elbs', ['account_id'], unique=False) - op.drop_column('destinations', 'plugin_name') - op.create_table('accounts', - sa.Column('id', sa.INTEGER(), server_default=sa.text(u"nextval('accounts_id_seq'::regclass)"), nullable=False), - sa.Column('account_number', sa.VARCHAR(length=32), autoincrement=False, nullable=True), - sa.Column('label', sa.VARCHAR(length=32), autoincrement=False, nullable=True), - sa.Column('notes', sa.TEXT(), autoincrement=False, nullable=True), - sa.PrimaryKeyConstraint('id', name=u'accounts_pkey'), - sa.UniqueConstraint('account_number', name=u'accounts_account_number_key'), - postgresql_ignore_search_path=False - ) - op.create_table('certificate_account_associations', - sa.Column('account_id', sa.INTEGER(), autoincrement=False, nullable=True), - sa.Column('certificate_id', sa.INTEGER(), autoincrement=False, nullable=True), - sa.ForeignKeyConstraint(['account_id'], [u'accounts.id'], name=u'certificate_account_associations_account_id_fkey', ondelete=u'CASCADE'), - sa.ForeignKeyConstraint(['certificate_id'], [u'certificates.id'], name=u'certificate_account_associations_certificate_id_fkey', ondelete=u'CASCADE') - ) - ### end Alembic commands ### diff --git a/lemur/migrations/versions/4bcfa2c36623_.py b/lemur/migrations/versions/4bcfa2c36623_.py deleted file mode 100644 index b7917f8ff7..0000000000 --- a/lemur/migrations/versions/4bcfa2c36623_.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Adding certificate signing algorithm - -Revision ID: 4bcfa2c36623 -Revises: 1ff763f5b80b -Create Date: 2015-10-06 10:03:47.993204 - -""" - -# revision identifiers, used by Alembic. -revision = '4bcfa2c36623' -down_revision = '1ff763f5b80b' - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.add_column('certificates', sa.Column('signing_algorithm', sa.String(length=128), nullable=True)) - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.drop_column('certificates', 'signing_algorithm') - ### end Alembic commands ### diff --git a/lemur/migrations/versions/4c8915e461b3_.py b/lemur/migrations/versions/4c8915e461b3_.py deleted file mode 100644 index f67d837ff5..0000000000 --- a/lemur/migrations/versions/4c8915e461b3_.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Adding notifications - -Revision ID: 4c8915e461b3 -Revises: 3b718f59b8ce -Create Date: 2015-07-24 14:34:57.316273 - -""" - -# revision identifiers, used by Alembic. -revision = '4c8915e461b3' -down_revision = '3b718f59b8ce' - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -import sqlalchemy_utils - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.create_table('notifications', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('label', sa.String(length=128), nullable=True), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('options', sqlalchemy_utils.types.json.JSONType(), nullable=True), - sa.Column('active', sa.Boolean(), nullable=True), - sa.Column('plugin_name', sa.String(length=32), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.drop_column(u'certificates', 'challenge') - op.drop_column(u'certificates', 'csr_config') - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.add_column(u'certificates', sa.Column('csr_config', sa.TEXT(), autoincrement=False, nullable=True)) - op.add_column(u'certificates', sa.Column('challenge', postgresql.BYTEA(), autoincrement=False, nullable=True)) - op.drop_table('notifications') - ### end Alembic commands ### diff --git a/lemur/migrations/versions/4dc5ddd111b8_.py b/lemur/migrations/versions/4dc5ddd111b8_.py deleted file mode 100644 index 8330744d79..0000000000 --- a/lemur/migrations/versions/4dc5ddd111b8_.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Creating a one-to-many relationship for notifications - -Revision ID: 4dc5ddd111b8 -Revises: 4c8915e461b3 -Create Date: 2015-07-24 15:02:04.398262 - -""" - -# revision identifiers, used by Alembic. -revision = '4dc5ddd111b8' -down_revision = '4c8915e461b3' - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.create_table('certificate_notification_associations', - sa.Column('notification_id', sa.Integer(), nullable=True), - sa.Column('certificate_id', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'), - sa.ForeignKeyConstraint(['notification_id'], ['notifications.id'], ondelete='cascade') - ) - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.drop_table('certificate_notification_associations') - ### end Alembic commands ### diff --git a/lemur/migrations/versions/ed422fc58ba_.py b/lemur/migrations/versions/ed422fc58ba_.py deleted file mode 100644 index d08a403b37..0000000000 --- a/lemur/migrations/versions/ed422fc58ba_.py +++ /dev/null @@ -1,255 +0,0 @@ -"""Migrates the private key encrypted column from AES to fernet encryption scheme. - -Revision ID: ed422fc58ba -Revises: 4bcfa2c36623 -Create Date: 2015-10-23 09:19:28.654126 - -""" -import base64 - -# revision identifiers, used by Alembic. -revision = 'ed422fc58ba' -down_revision = '4bcfa2c36623' -import six - -from StringIO import StringIO - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.sql import text - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.fernet import Fernet, MultiFernet - -from flask import current_app -from lemur.common.utils import get_psuedo_random_string - -conn = op.get_bind() - -op.drop_table('encrypted_keys') -op.drop_table('encrypted_passwords') - -# helper tables to migrate data -temp_key_table = op.create_table('encrypted_keys', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('aes', sa.Binary()), - sa.Column('fernet', sa.Binary()), - sa.PrimaryKeyConstraint('id') - ) - -# helper table to migrate data -temp_password_table = op.create_table('encrypted_passwords', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('aes', sa.Binary()), - sa.Column('fernet', sa.Binary()), - sa.PrimaryKeyConstraint('id') - ) - - -# From http://sqlalchemy-utils.readthedocs.org/en/latest/_modules/sqlalchemy_utils/types/encrypted.html#EncryptedType -# for migration purposes only -class EncryptionDecryptionBaseEngine(object): - """A base encryption and decryption engine. - - This class must be sub-classed in order to create - new engines. - """ - - def _update_key(self, key): - if isinstance(key, six.string_types): - key = key.encode() - digest = hashes.Hash(hashes.SHA256(), backend=default_backend()) - digest.update(key) - engine_key = digest.finalize() - - self._initialize_engine(engine_key) - - def encrypt(self, value): - raise NotImplementedError('Subclasses must implement this!') - - def decrypt(self, value): - raise NotImplementedError('Subclasses must implement this!') - - -class AesEngine(EncryptionDecryptionBaseEngine): - """Provide AES encryption and decryption methods.""" - - BLOCK_SIZE = 16 - PADDING = six.b('*') - - def _initialize_engine(self, parent_class_key): - self.secret_key = parent_class_key - self.iv = self.secret_key[:16] - self.cipher = Cipher( - algorithms.AES(self.secret_key), - modes.CBC(self.iv), - backend=default_backend() - ) - - def _pad(self, value): - """Pad the message to be encrypted, if needed.""" - BS = self.BLOCK_SIZE - P = self.PADDING - padded = (value + (BS - len(value) % BS) * P) - return padded - - def encrypt(self, value): - if not isinstance(value, six.string_types): - value = repr(value) - if isinstance(value, six.text_type): - value = str(value) - value = value.encode() - value = self._pad(value) - encryptor = self.cipher.encryptor() - encrypted = encryptor.update(value) + encryptor.finalize() - encrypted = base64.b64encode(encrypted) - return encrypted - - def decrypt(self, value): - if isinstance(value, six.text_type): - value = str(value) - decryptor = self.cipher.decryptor() - decrypted = base64.b64decode(value) - decrypted = decryptor.update(decrypted) + decryptor.finalize() - decrypted = decrypted.rstrip(self.PADDING) - if not isinstance(decrypted, six.string_types): - decrypted = decrypted.decode('utf-8') - return decrypted - - -def migrate_to_fernet(aes_encrypted, old_key, new_key): - """ - Will attempt to migrate an aes encrypted to fernet encryption - :param aes_encrypted: - :return: fernet encrypted value - """ - engine = AesEngine() - engine._update_key(old_key) - - if not isinstance(aes_encrypted, six.string_types): - return - - aes_decrypted = engine.decrypt(aes_encrypted) - fernet_encrypted = MultiFernet([Fernet(k) for k in new_key]).encrypt(bytes(aes_decrypted)) - - # sanity check - fernet_decrypted = MultiFernet([Fernet(k) for k in new_key]).decrypt(fernet_encrypted) - if fernet_decrypted != aes_decrypted: - raise Exception("WARNING: Decrypted values do not match!") - - return fernet_encrypted - - -def migrate_from_fernet(fernet_encrypted, old_key, new_key): - """ - Will attempt to migrate from a fernet encryption to aes - :param fernet_encrypted: - :return: - """ - engine = AesEngine() - engine._update_key(new_key) - - fernet_decrypted = MultiFernet([Fernet(k) for k in old_key]).decrypt(fernet_encrypted) - aes_encrypted = engine.encrypt(fernet_decrypted) - - # sanity check - aes_decrypted = engine.decrypt(aes_encrypted) - if fernet_decrypted != aes_decrypted: - raise Exception("WARNING: Decrypted values do not match!") - - return aes_encrypted - - -def upgrade(): - old_key = current_app.config.get('LEMUR_ENCRYPTION_KEY') - print "Using: {0} as decryption key".format(old_key) - # generate a new fernet token - - if current_app.config.get('LEMUR_ENCRYPTION_KEYS'): - new_key = current_app.config.get('LEMUR_ENCRYPTION_KEYS') - else: - new_key = [Fernet.generate_key()] - - print "Using: {0} as new encryption key, save this and place it in your configuration!".format(new_key) - - # migrate private_keys - temp_keys = [] - for id, private_key in conn.execute(text('select id, private_key from certificates where private_key is not null')): - aes_encrypted = StringIO(private_key).read() - fernet_encrypted = migrate_to_fernet(aes_encrypted, old_key, new_key) - temp_keys.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted}) - - op.bulk_insert(temp_key_table, temp_keys) - - for id, fernet in conn.execute(text('select id, fernet from encrypted_keys')): - stmt = text("update certificates set private_key=:key where id=:id") - stmt = stmt.bindparams(key=fernet, id=id) - op.execute(stmt) - print "Certificate {0} has been migrated".format(id) - - # migrate role_passwords - temp_passwords = [] - for id, password in conn.execute(text('select id, password from roles where password is not null')): - aes_encrypted = StringIO(password).read() - fernet_encrypted = migrate_to_fernet(aes_encrypted, old_key, new_key) - temp_passwords.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted}) - - op.bulk_insert(temp_password_table, temp_passwords) - - for id, fernet in conn.execute(text('select id, fernet from encrypted_passwords')): - stmt = text("update roles set password=:password where id=:id") - stmt = stmt.bindparams(password=fernet, id=id) - print stmt - op.execute(stmt) - print "Password {0} has been migrated".format(id) - - op.drop_table('encrypted_keys') - op.drop_table('encrypted_passwords') - - -def downgrade(): - old_key = current_app.config.get('LEMUR_ENCRYPTION_KEYS') - print "Using: {0} as decryption key(s)".format(old_key) - - # generate aes valid key - if current_app.config.get('LEMUR_ENCRYPTION_KEY'): - new_key = current_app.config.get('LEMUR_ENCRYPTION_KEY') - else: - new_key = get_psuedo_random_string() - print "Using: {0} as the encryption key, save this and place it in your configuration!".format(new_key) - - # migrate keys - temp_keys = [] - for id, private_key in conn.execute(text('select id, private_key from certificates where private_key is not null')): - fernet_encrypted = StringIO(private_key).read() - aes_encrypted = migrate_from_fernet(fernet_encrypted, old_key, new_key) - temp_keys.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted}) - - op.bulk_insert(temp_key_table, temp_keys) - - for id, aes in conn.execute(text('select id, aes from encrypted_keys')): - stmt = text("update certificates set private_key=:key where id=:id") - stmt = stmt.bindparams(key=aes, id=id) - print stmt - op.execute(stmt) - print "Certificate {0} has been migrated".format(id) - - # migrate role_passwords - temp_passwords = [] - for id, password in conn.execute(text('select id, password from roles where password is not null')): - fernet_encrypted = StringIO(password).read() - aes_encrypted = migrate_from_fernet(fernet_encrypted, old_key, new_key) - temp_passwords.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted}) - - op.bulk_insert(temp_password_table, temp_passwords) - - for id, aes in conn.execute(text('select id, aes from encrypted_passwords')): - stmt = text("update roles set password=:password where id=:id") - stmt = stmt.bindparams(password=aes, id=id) - op.execute(stmt) - print "Password {0} has been migrated".format(id) - - op.drop_table('encrypted_keys') - op.drop_table('encrypted_passwords')