Merge pull request #166 from mozilla-services/feat/165

feat: add dynamic_settings table for on-the-fly ops config

tokenserver/assignment/sqlnode/migrations/versions/5d056c5b8f57_create_dyn_settings_table.py

@@ -0,0 +1,27 @@
+"""create dyn_settings table
+
+Revision ID: 5d056c5b8f57
+Revises: 75e8ca84b0bc
+Create Date: 2020-01-06 08:16:15.546054
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '5d056c5b8f57'
+down_revision = '75e8ca84b0bc'
+
+
+def upgrade():
+    op.create_table(
+        'dynamic_settings',
+        sa.Column('setting', sa.String(100), primary_key=True),
+        sa.Column('value', sa.String(255), nullable=False),
+        sa.Column('description', sa.String(255))
+    )
+
+
+def downgrade():
+    op.drop_table('dynamic_settings')
+    pass

tokenserver/assignment/sqlnode/schemas.py

@@ -116,3 +116,17 @@ def __table_args__(cls):
 
 
 _add('nodes', _NodesBase)
+
+
+class _SettingsBase(object):
+    """This table holds dynamic setting values for operations.
+
+    Really, only useful for longer lived SQL nodes right now.
+
+    """
+    setting = Column(String(100), primary_key=True, nullable=False)
+    value = Column(String(255), nullable=False)
+    description = Column(String(255), nullable=True)
+
+
+_add('dynamic_settings', _SettingsBase)

tokenserver/assignment/sqlnode/sql.py

@@ -11,6 +11,7 @@
 import math
 import traceback
 import hashlib
+import time
 from mozsvc.exceptions import BackendError
 
 from sqlalchemy.sql import select, update, and_
@@ -184,6 +185,18 @@
 """)
 
 
+_GET_DYNAMIC_SETTING = sqltext("""
+select
+    value
+from
+    dynamic_settings
+where
+    setting = :setting
+""")
+
+MIGRATION_CACHE_LIFESPAN = 300
+
+
 class SQLNodeAssignment(object):
 
     implements(INodeAssignment)
@@ -194,6 +207,7 @@ def __init__(self, sqluri, create_tables=False, pool_size=100,
                  spanner_node_id=None, migrate_new_user_percentage=0,
                  **kw):
         self._cached_service_ids = {}
+        self._migration_percentage_cache_ttl = 0
         self.sqluri = sqluri
         if pool_reset_on_return.lower() in ('', 'none'):
             pool_reset_on_return = None
@@ -236,8 +250,10 @@ def __init__(self, sqluri, create_tables=False, pool_size=100,
         self.services = get_cls('services', _Base)
         self.nodes = get_cls('nodes', _Base)
         self.users = get_cls('users', _Base)
+        self.dyn_settings = get_cls('dynamic_settings', _Base)
 
-        for table in (self.services, self.nodes, self.users):
+        for table in (self.services, self.nodes,
+                      self.users, self.dyn_settings):
             table.metadata.bind = self._engine
             if create_tables:
                 table.create(checkfirst=True)
@@ -316,13 +332,36 @@ def get_user(self, service, email):
         finally:
             res.close()
 
+    def get_migration_percent(self):
+        """get a cached Ops controllable percentage value for the number of
+        new users to migrate to Spanner."""
+        if self._migration_percentage_cache_ttl > time.time():
+            return self.migrate_new_user_percentage
+        default = self.migrate_new_user_percentage or 0
+        try:
+            res = self._safe_execute(
+                _GET_DYNAMIC_SETTING,
+                {"setting": "migrate_new_user_percentage"})
+            self.migrate_new_user_percentage = (
+                int(res.fetchone()[0]) or default
+            )
+            self._migration_percentage_cache_ttl = \
+                time.time() + MIGRATION_CACHE_LIFESPAN
+        except Exception as ex:
+            logger.warn(
+                "Could not get migration percent \"{}\" using default: {}"
+                .format(ex, default)
+            )
+        return self.migrate_new_user_percentage
+
     def should_allocate_to_spanner(self, email):
         """use a simple, reproducable hashing mechanism to determine if
         a user should be provisioned to spanner. Does not need to be
         secure, just a selectable percentage."""
-        if self.migrate_new_user_percentage:
+        migrate = self.get_migration_percent()
+        if migrate:
             pick = ord(hashlib.sha1(email.encode()).digest()[0])
-            return pick < (256 * (self.migrate_new_user_percentage * .01))
+            return pick < (256 * (migrate * .01))
         else:
             return False
 

tokenserver/assignment/sqlnode/sqliteschemas.py

@@ -6,8 +6,10 @@
 """
 from tokenserver.assignment.sqlnode.schemas import (_UsersBase,
                                                     _NodesBase,
+                                                    _SettingsBase,
                                                     Integer,
                                                     Column,
+                                                    String,
                                                     _add,
                                                     declared_attr)
 
@@ -37,3 +39,14 @@ def __table_args__(cls):
 
 
 _add('users', _SQLITEUsersBase)
+
+
+class _SQLITESettingsBase(_SettingsBase):
+    setting = Column(String(100), primary_key=True)
+
+    @declared_attr
+    def __table_args__(cls):
+        return()
+
+
+_add('dynamic_settings', _SQLITESettingsBase)