From 4217cd97574398b46075b35ff447a2c8d2ea2340 Mon Sep 17 00:00:00 2001 From: Jonathan Freedman Date: Thu, 29 Jun 2017 11:28:54 -0700 Subject: [PATCH 1/3] handle 0.6.x style secret backend sub-path error --- aomi/model/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aomi/model/__init__.py b/aomi/model/__init__.py index 42eb834..7832830 100644 --- a/aomi/model/__init__.py +++ b/aomi/model/__init__.py @@ -256,7 +256,11 @@ def filtered(self): def read(self, client): """Read from Vault while handling non surprising errors.""" log("Reading from %s" % self, self.opt) - return client.read(self.path) + try: + return client.read(self.path) + except hvac.exceptions.InvalidRequest as vault_exception: + if vault_exception.message.startswith('no handler for route'): + return None @wrap_vault("writing") def write(self, client): From 56e7b5030f8e9d447d29aed80137257503f8572e Mon Sep 17 00:00:00 2001 From: Jonathan Freedman Date: Tue, 11 Jul 2017 22:32:56 -0700 Subject: [PATCH 2/3] proper ordering on removing mounts --- aomi/model/__init__.py | 70 +++++++++++++++++++++++++++--------------- docs/aws.md | 2 +- docs/secretfile.md | 2 ++ 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/aomi/model/__init__.py b/aomi/model/__init__.py index 7832830..02795c2 100644 --- a/aomi/model/__init__.py +++ b/aomi/model/__init__.py @@ -76,7 +76,7 @@ def ensure_backend(resource, backend, backends, opt): if backend == LogBackend: new_mount = backend(resource, opt) else: - new_mount = backend(resource.mount, resource.backend, opt) + new_mount = backend(resource, opt) backends.append(new_mount) return new_mount @@ -271,7 +271,11 @@ def write(self, client): def delete(self, client): """Delete from Vault while handling non-surprising errors.""" log("Deleting %s" % self, self.opt) - client.delete(self.path) + try: + client.delete(self.path) + except hvac.exceptions.InvalidPath as vault_exception: + if vault_exception.message.startswith('no handler for route'): + return None class Secret(Resource): @@ -390,23 +394,32 @@ def sync(self, vault_client): has the effect of updating every resource which is in the context.""" active_mounts = [] - for mount in self.mounts(): - if not mount.existing: - mount.sync(vault_client) for auth in self.auths(): - if not auth.existing: - auth.sync(vault_client) - for blog in self.logs(): - if not blog.existing: - blog.sync(vault_client) - for resource in self.resources(): - if isinstance(resource, (Secret, Mount)) and resource.present: - active_mount = find_backend(resource.mount, active_mounts) - if not active_mount: - actual_mount = find_backend(resource.mount, self._mounts) - if actual_mount: - active_mounts.append(actual_mount) - + auth.sync(vault_client) + for audit_log in self.logs(): + audit_log.sync(vault_client) + + # Handle mounts only on the first pass. This allows us to + # ensure that everything is in order prior to actually + # provisioning secrets. Note we handle removals before + # anything else, allowing us to address mount conflicts. + mounts = [x for x in self.resources() + if isinstance(x, (Secret, Mount))] + s_resources = sorted(mounts, cmp=lambda x, y: + cmp(x.present, y.present)) + + for resource in s_resources: + active_mount = find_backend(resource.mount, active_mounts) + if not active_mount: + actual_mount = find_backend(resource.mount, self._mounts) + active_mounts.append(actual_mount) + actual_mount.sync(vault_client) + + # Now handle everything else. If "best practices" are being + # adhered to then every generic mountpoint should exist by now + not_mounts = [x for x in self.resources() + if not isinstance(x, (Mount))] + for resource in not_mounts: resource.sync(vault_client) for mount in self.mounts(): @@ -448,6 +461,7 @@ class Mount(Resource): required_fields = ['path'] config_key = 'mounts' backend = 'generic' + secret_format = 'mount point' def __init__(self, obj, opt): super(Mount, self).__init__(obj, opt) @@ -482,22 +496,30 @@ def __str__(self): return "%s %s" % (self.backend, self.path) - def __init__(self, path, backend, opt): - self.path = sanitize_mount(path) - self.backend = backend + def __init__(self, resource, opt): + self.path = sanitize_mount(resource.mount) + self.backend = resource.backend self.existing = False + self.present = resource.present self.opt = opt def sync(self, vault_client): """Synchronizes the local and remote Vault resources. Has the net effect of adding backend if needed""" - if not self.existing: + if not self.existing and self.present: self.actually_mount(vault_client) log("Mounting %s backend on %s" % (self.backend, self.path), self.opt) - else: + elif self.existing and self.present: log("%s backend already mounted on %s" % (self.backend, self.path), self.opt) + elif self.existing and not self.present: + self.unmount(vault_client) + log("Unmounting %s backend on %s" % + (self.backend, self.path), self.opt) + elif not self.existing and not self.present: + log("%s backend already unmounted on %s" % + (self.backend, self.path), self.opt) def fetch(self, backends): """Updates local resource with context on whether this @@ -506,8 +528,6 @@ def fetch(self, backends): def unmount(self, client): """Unmounts a backend within Vault""" - log("Unmounting %s backend from %s" % - (self.backend, self.path), self.opt) getattr(client, self.unmount_fun)(mount_point=self.path) def actually_mount(self, client): diff --git a/docs/aws.md b/docs/aws.md index 467ca24..ae3e8ca 100644 --- a/docs/aws.md +++ b/docs/aws.md @@ -9,7 +9,7 @@ The aomi tool is able to write to the AWS backend of Vault. By specifying an appropriately populated `aws_file` you can create [AWS secret backends](https://www.vaultproject.io/docs/secrets/aws/index.html) in Vault. The `aws_file` must point to a valid file, and the base of the AWS credentials will be set by the `mount`. -The AWS file contains the `access_key_id`, and `secret_access_key`. The `region`, and a list of AWS roles that will be loaded by Vault are in the `Secretfile`. Note that you may specify either an inline `policy` _or_ a native AWS `arn`. The `name` of each role will be used to compute the final path for accessing credentials. The policy files are simply JSON IAM Access representations. The following example would create an AWS Vault secret backend at `foo/bar/baz` based on the account and policy information defined in `.secrets/aws.yml`. While `lease` and `lease_max` are provided in this example, they are not strictly required. Note that you can also specify a `state` as either `present` (the default) or `absent`. +The AWS file contains the `access_key_id`, and `secret_access_key`. The `region`, and a list of AWS roles that will be loaded by Vault are in the `Secretfile`. Note that you may specify either an inline `policy` _or_ a native AWS `arn`. The `name` of each role will be used to compute the final path for accessing credentials. The policy files are simply JSON IAM Access representations. The following example would create an AWS Vault secret backend at `foo/bar/baz` based on the account and policy information defined in `.secrets/aws.yml`. While `lease` and `lease_max` are provided in this example, they are not strictly required. Note that you can specify the `state` as either `absent` or `present` for each individual role. Note that a previous version had `lease`, `lease_max`, `region`, and the `roles` section located in the `aws_file` itself - this behavior is now considered deprecated. The _only_ thing which should be present in the AWS yaml is the actual secrets. diff --git a/docs/secretfile.md b/docs/secretfile.md index e1d9e08..9d88764 100644 --- a/docs/secretfile.md +++ b/docs/secretfile.md @@ -43,6 +43,8 @@ The Secretfile is interpreted as a YAML file. Prior to parsing, aomi will render * Vault policies and audit logs are also configurable. These do not have any secrets associated with them. * You can define some metadata which is limited to GPG/Keybase information, used for cold storage of secrets. +It is possible to be explict about the presence of a Vault construct on the server. Every entry should support the `state` value, which can be set to either `present` (the default) or `absent`. + # Tagging Every entry which will affect Vault may be "tagged". Any and all tags must be referenced in order for the resource to be processed. Untagged resources will only be processed if tags are not specified on the command line. The following example shows two sets of static files, each tied to a different tag. This is one way of having a single `Secretfile` which can be used to populate multiple environments. From d696cd2a0215d47ff63270c1f0d31af4fa8fad2b Mon Sep 17 00:00:00 2001 From: Jonathan Freedman Date: Wed, 12 Jul 2017 17:36:32 -0700 Subject: [PATCH 3/3] Account for Python3 --- aomi/model/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/aomi/model/__init__.py b/aomi/model/__init__.py index 02795c2..5714a51 100644 --- a/aomi/model/__init__.py +++ b/aomi/model/__init__.py @@ -14,6 +14,13 @@ from aomi.validation import sanitize_mount, check_obj +def absent_sort(resource): + """Used to sort resources in a way where things that + are being removed are prioritized over things that + are being added or modified""" + return resource.present + + def py_resources(): """Discovers all aomi Vault resource models""" aomi_mods = [m for @@ -259,7 +266,7 @@ def read(self, client): try: return client.read(self.path) except hvac.exceptions.InvalidRequest as vault_exception: - if vault_exception.message.startswith('no handler for route'): + if str(vault_exception).startswith('no handler for route'): return None @wrap_vault("writing") @@ -274,7 +281,7 @@ def delete(self, client): try: client.delete(self.path) except hvac.exceptions.InvalidPath as vault_exception: - if vault_exception.message.startswith('no handler for route'): + if str(vault_exception).startswith('no handler for route'): return None @@ -405,8 +412,8 @@ def sync(self, vault_client): # anything else, allowing us to address mount conflicts. mounts = [x for x in self.resources() if isinstance(x, (Secret, Mount))] - s_resources = sorted(mounts, cmp=lambda x, y: - cmp(x.present, y.present)) + + s_resources = sorted(mounts, key=absent_sort) for resource in s_resources: active_mount = find_backend(resource.mount, active_mounts)