Support for docker secrets and referenced vars

environ/environ.py

@@ -115,6 +115,7 @@ class Env(object):
         "xapian": "haystack.backends.xapian_backend.XapianEngine",
         "simple": "haystack.backends.simple_backend.SimpleEngine",
     }
+    DOCKER_SECRETS_DIR = '/run/secrets/'
 
     def __init__(self, **scheme):
         self.scheme = scheme
@@ -233,6 +234,24 @@ def path(self, var, default=NOTSET, **kwargs):
         """
         return Path(self.get_value(var, default=default), **kwargs)
 
+    def get_docker_secret(self,secret_name):
+
+        secret = ''
+        secret_filepath = os.path.join(self.DOCKER_SECRETS_DIR,secret_name)
+        if not os.path.exists(secret_filepath):
+            warnings.warn("%s doesn't exist - check that your Docker node "
+                          "is correctly provisioned with secrets." % secret_filepath)
+            return secret
+
+        with open(secret_filepath, 'r') as fl:
+            try:
+                secret = fl.readline().strip()
+            except IOError:
+                warnings.warn(
+                    "Error reading %s - if you're not configuring your "
+                    "environment separately, check this." % secret_name)
+        return secret
+
     def get_value(self, var, cast=None, default=NOTSET, parse_default=False):
         """Return value for given environment variable.
 
@@ -283,6 +302,16 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False):
             value = value.lstrip('$')
             value = self.get_value(value, cast=cast, default=default)
 
+        # Substitute any referenced variables in the string
+        if value is not None and type(value) is str:
+            for match in re.finditer(r'\{\{([A-Za-z_0-9]+)\}\}',value):
+                value = value.replace(match.group(0),self.get_value(match.group(1), cast=cast, default=default))
+
+        # Read the values of docker-secrets
+        if value is not None and type(value) is str:
+            for match in re.finditer(r'\{\{DOCKER-SECRET:([A-Za-z_0-9]+)\}\}', value):
+                value = value.replace(match.group(0), self.get_docker_secret(match.group(1)))
+
         if cast is None and default is not None and not isinstance(default, NoValue):
             cast = type(default)
 
@@ -657,6 +686,38 @@ def read_env(cls, env_file=None, **overrides):
         for key, value in overrides.items():
             cls.ENVIRON.setdefault(key, value)
 
+    @classmethod
+    def read_docker_secrets(cls, secrets_dir='/run/secrets/', **overrides):
+        """
+        Read the files in the Docker Secrets folder into the environment
+
+        The secrets folder is a dict-like structure where for each file:
+            key = filename
+            value = first line of the file
+
+        There is currently no support for dynamically updating individual
+        secrets; the whole application must be reloaded for secrets to be
+        updated.
+        """
+
+        if not os.path.exists(secrets_dir):
+            warnings.warn("%s doesn't exist - check that your Docker node "
+                          "is correctly provisioned with secrets." % secrets_dir)
+            return
+
+        for secret_filename in os.listdir(secrets_dir):
+            with open(os.path.join(secrets_dir, secret_filename), 'r') as fl:
+                try:
+                    cls.ENVIRON.setdefault(os.path.splitext(secret_filename.upper())[0], fl.readline().strip())
+                except IOError:
+                    warnings.warn(
+                        "Error reading %s - if you're not configuring your "
+                        "environment separately, check this." % secret_filename)
+
+        # set defaults
+        for key, value in overrides.items():
+            cls.ENVIRON.setdefault(key, value)
+
 
 class Path(object):
 

environ/run/secrets/docker_secret_1

@@ -0,0 +1 @@
+secret1

environ/run/secrets/docker_secret_2

@@ -0,0 +1 @@
+secret2

environ/test.py

@@ -10,11 +10,12 @@
 
 from environ import Env, Path
 
-
 class BaseTests(unittest.TestCase):
 
     URL = 'http://www.google.com/'
     POSTGRES = 'postgres://uf07k1:[email protected]:5431/d8r82722'
+    POSTGRES_WITH_VARS = 'postgres://{{USER}}:{{PASSWORD}}@ec2-107-21-253-135.compute-1.amazonaws.com:5431/d8r82722'
+    POSTGRES_WITH_DOCKER_SECRETS = 'postgres://{{DOCKER-SECRET:docker_secret_1}}:{{DOCKER-SECRET:docker_secret_2}}@ec2-107-21-253-135.compute-1.amazonaws.com:5431/d8r82722'
     MYSQL = 'mysql://bea6eb0:[email protected]/heroku_97681?reconnect=true'
     MYSQLGIS = 'mysqlgis://user:[email protected]/some_database'
     SQLITE = 'sqlite:////full/path/to/your/database/file.sqlite'
@@ -44,12 +45,19 @@ def generateData(cls):
                     BOOL_FALSE_VAR='0',
                     BOOL_FALSE_VAR2='False',
                     PROXIED_VAR='$STR_VAR',
+                    PROXIED_VAR_2='{{STR_VAR}}',
                     INT_LIST='42,33',
                     INT_TUPLE='(42,33)',
                     STR_LIST_WITH_SPACES=' foo,  bar',
                     EMPTY_LIST='',
                     DICT_VAR='foo=bar,test=on',
                     DATABASE_URL=cls.POSTGRES,
+                    USER='uf07k1',
+                    PASSWORD='wegauwhg',
+                    DOCKER_SECRET_3='{{DOCKER-SECRET:docker_secret_1}}',
+                    DOCKER_SECRET_4='{{DOCKER-SECRET:docker_secret_2}}',
+                    DATABASE_URL_WITH_VARS=cls.POSTGRES_WITH_VARS,
+                    DATABASE_URL_WITH_DOCKER_SECRETS=cls.POSTGRES_WITH_DOCKER_SECRETS,
                     DATABASE_MYSQL_URL=cls.MYSQL,
                     DATABASE_MYSQL_GIS_URL=cls.MYSQLGIS,
                     DATABASE_SQLITE_URL=cls.SQLITE,
@@ -69,6 +77,7 @@ def setUp(self):
         self._old_environ = os.environ
         os.environ = Env.ENVIRON = self.generateData()
         self.env = Env()
+        self.env.DOCKER_SECRETS_DIR = Path(__file__, is_file=True)('run/secrets/')
 
     def tearDown(self):
         os.environ = self._old_environ
@@ -127,6 +136,11 @@ def test_bool_false(self):
 
     def test_proxied_value(self):
         self.assertEqual('bar', self.env('PROXIED_VAR'))
+        self.assertEqual('bar', self.env('PROXIED_VAR_2'))
+
+    def test_docker_secrets(self):
+        self.assertEqual('secret1', self.env('DOCKER_SECRET_3'))
+        self.assertEqual('secret2', self.env('DOCKER_SECRET_4'))
 
     def test_int_list(self):
         self.assertTypeAndValue(list, [42, 33], self.env('INT_LIST', cast=[int]))
@@ -150,7 +164,6 @@ def test_dict_value(self):
         self.assertTypeAndValue(dict, self.DICT, self.env.dict('DICT_VAR'))
 
     def test_dict_parsing(self):
-
         self.assertEqual({'a': '1'}, self.env.parse_value('a=1', dict))
         self.assertEqual({'a': 1}, self.env.parse_value('a=1', dict(value=int)))
         self.assertEqual({'a': ['1', '2', '3']}, self.env.parse_value('a=1,2,3', dict(value=[str])))
@@ -182,6 +195,14 @@ def test_db_url_value(self):
         self.assertEqual(pg_config['PASSWORD'], 'wegauwhg')
         self.assertEqual(pg_config['PORT'], 5431)
 
+        pg_config_with_vars = self.env.db('DATABASE_URL_WITH_VARS')
+        self.assertEqual(pg_config_with_vars['USER'], 'uf07k1')
+        self.assertEqual(pg_config_with_vars['PASSWORD'], 'wegauwhg')
+
+        pg_config_with_docker_secrets = self.env.db('DATABASE_URL_WITH_DOCKER_SECRETS')
+        self.assertEqual(pg_config_with_docker_secrets['USER'], 'secret1')
+        self.assertEqual(pg_config_with_docker_secrets['PASSWORD'], 'secret2')
+
         mysql_config = self.env.db('DATABASE_MYSQL_URL')
         self.assertEqual(mysql_config['ENGINE'], 'django.db.backends.mysql')
         self.assertEqual(mysql_config['NAME'], 'heroku_97681')
@@ -282,8 +303,14 @@ def setUp(self):
         super(FileEnvTests, self).setUp()
         Env.ENVIRON = {}
         self.env = Env()
+        self.env.DOCKER_SECRETS_DIR = Path(__file__, is_file=True)('run/secrets/')
         file_path = Path(__file__, is_file=True)('test_env.txt')
         self.env.read_env(file_path, PATH_VAR=Path(__file__, is_file=True).__root__)
+        self.env.read_docker_secrets(self.env.DOCKER_SECRETS_DIR)
+
+    def test_load_docker_secrets(self):
+        self.assertEqual('secret1', self.env('DOCKER_SECRET_1'))
+        self.assertEqual('secret2', self.env('DOCKER_SECRET_2'))
 
 class SubClassTests(EnvTests):
 
@@ -293,6 +320,7 @@ def setUp(self):
         class MyEnv(Env):
             ENVIRON = self.CONFIG
         self.env = MyEnv()
+        self.env.DOCKER_SECRETS_DIR = Path(__file__, is_file=True)('run/secrets/')
 
     def test_singleton_environ(self):
         self.assertTrue(self.CONFIG is self.env.ENVIRON)

environ/test_env.txt

@@ -13,11 +13,16 @@ DATABASE_SQLITE_URL=sqlite:////full/path/to/your/database/file.sqlite
 JSON_VAR={"three": 33.44, "two": 2, "one": "bar"}
 BOOL_TRUE_VAR=1
 DATABASE_URL=postgres://uf07k1:[email protected]:5431/d8r82722
+USER=uf07k1
+PASSWORD=wegauwhg
+DATABASE_URL_WITH_VARS=postgres://{{USER}}:{{PASSWORD}}@ec2-107-21-253-135.compute-1.amazonaws.com:5431/d8r82722
+DATABASE_URL_WITH_DOCKER_SECRETS=postgres://{{DOCKER-SECRET:docker_secret_1}}:{{DOCKER-SECRET:docker_secret_2}}@ec2-107-21-253-135.compute-1.amazonaws.com:5431/d8r82722
 FLOAT_VAR=33.3
 FLOAT_COMMA_VAR=33,3
 FLOAT_STRANGE_VAR1=123,420,333.3
 FLOAT_STRANGE_VAR2=123.420.333,3
 PROXIED_VAR=$STR_VAR
+PROXIED_VAR_2={{STR_VAR}}
 EMPTY_LIST=
 INT_VAR=42
 STR_LIST_WITH_SPACES= foo,  bar
@@ -30,4 +35,6 @@ DATABASE_ORACLE_TNS_URL=oracle://user:password@sid
 DATABASE_ORACLE_URL=oracle://user:password@host:1521/sid
 DATABASE_REDSHIFT_URL=redshift://user:password@examplecluster.abc123xyz789.us-west-2.redshift.amazonaws.com:5439/dev
 DATABASE_CUSTOM_BACKEND_URL=custom.backend://user:[email protected]:5430/database
+DOCKER_SECRET_3={{DOCKER-SECRET:docker_secret_1}}
+DOCKER_SECRET_4={{DOCKER-SECRET:docker_secret_2}}
 export EXPORTED_VAR="exported var"