Skip to content

Commit

Permalink
chore: Add pre-commit, fix types (#15)
Browse files Browse the repository at this point in the history
  • Loading branch information
edgarrmondragon authored Feb 21, 2023
1 parent eac1167 commit cd44fd7
Show file tree
Hide file tree
Showing 12 changed files with 274 additions and 101 deletions.
2 changes: 1 addition & 1 deletion .env
Original file line number Diff line number Diff line change
@@ -1 +1 @@
EVIDENCE_HOME=sample/
EVIDENCE_HOME=sample/
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -160,4 +160,4 @@ cython_debug/
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
#.idea/
41 changes: 41 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
ci:
autofix_prs: true
autoupdate_schedule: weekly
autoupdate_commit_msg: 'chore: pre-commit autoupdate'

repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-json
- id: check-toml
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace

- repo: https://github.com/psf/black
rev: 23.1.0
hooks:
- id: black

- repo: https://github.com/pycqa/isort
rev: 5.12.0
hooks:
- id: isort

- repo: https://github.com/asottile/pyupgrade
rev: v3.3.1
hooks:
- id: pyupgrade
args: [--py37-plus]

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.0.1
hooks:
- id: mypy

- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: v0.0.249
hooks:
- id: ruff
args: ["--fix"]
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# evidence-ext

[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v1.json)](https://github.com/charliermarsh/ruff)
[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/MeltanoLabs/evidence-ext/main.svg)](https://results.pre-commit.ci/latest/github/MeltanoLabs/evidence-ext/main)

`evidence-ext` is A Meltano utility extension for [Evidence.dev](https://evidence.dev) 📊

## Testing with Meltano
Expand Down
114 changes: 81 additions & 33 deletions evidence_ext/config.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
"""This is currently only used to suppress the config file.
"""Configuration management for Evidence.
This is currently only used to suppress the config file.
Evidence tries to read the config file first, therefore we must remove it
in order to reliably pass settings through as env vars from Meltano.
Expand All @@ -8,18 +10,47 @@
not accessible via env vars (which it doesn't seem like there are).
"""

from __future__ import annotations

import contextlib
import json
import os
import typing as t
from pathlib import Path

if t.TYPE_CHECKING:
from typing import Iterable, Iterator


class MissingEnvVarError(Exception):
"""Missing env var."""

def __init__(self, env_var: str, database: str | None) -> None:
"""Initialize exception.
Args:
env_var: Name of the missing env var.
database: Name of the database.
"""
msg = f"Environment variable '{env_var}' required for database type"

if database:
msg = f"{msg} '{database}'"

super().__init__(msg)


class EvidenceConfig:
"""Evidence Config."""

def __init__(self, evidence_home):
def __init__(self, evidence_home: str) -> None:
"""Initialize Evidence configuation.
Args:
evidence_home: Path to the Evidence home directory.
"""
self.evidence_home = evidence_home
self.config_file = (
self._config_file = (
Path(self.evidence_home)
/ ".evidence"
/ "template"
Expand All @@ -28,85 +59,98 @@ def __init__(self, evidence_home):
self.database = os.environ.get("EVIDENCE_DATABASE")

@contextlib.contextmanager
def suppress_config_file(self):
def suppress_config_file(self) -> Iterator[None]:
"""Suppress Evidence config file.
As evidence checks its config file _before_ env vars,
we need to remove it before run and replace it after (if it exists).
"""
config = None
if self.config_file.exists():
with self.config_file.open("r") as cfg:
if self._config_file.exists():
with self._config_file.open("r") as cfg:
config = json.load(cfg)
self._cleanup_config()
try:
yield
finally:
if config:
with self.config_file.open("w", encoding="utf-8") as cfg:
with self._config_file.open("w", encoding="utf-8") as cfg:
json.dump(config, cfg)

@contextlib.contextmanager
def config_file(self):
def config_file(self) -> Iterator[None]:
"""Context manager for JIT creation of Evidence config file."""
self._write_config()
yield
self._cleanup_config()

def get_config(self):
"""Read config from Env Vars, validating by database type."""
def get_config(self) -> dict:
"""Read config from Env Vars, validating by database type.
Returns:
dict: Evidence configuation.
Raises:
KeyError: If database type is not supported.
"""
if not self.database:
return {}

if self.database in ("duckdb", "sqlite"):
return self._get_config_duckdb_sqlite()
elif self.database == "bigquery":
if self.database == "bigquery":
return self._get_config_bigquery()
elif self.database == "mysql":
if self.database == "mysql":
return self._get_config_mysql()
else:
raise KeyError(
f"Database connection {self.database} is not yet supported by evidence-ext."
)

def _write_config(self):
msg = (
f"Database connection {self.database} is not yet supported by evidence-ext."
)
raise KeyError(msg)

def _write_config(self) -> None:
"""Write Evidence config from env vars."""
self.config_file.parent.mkdir(parents=True, exist_ok=True)
config = self._get_config()
with self.config_file.open("w", encoding="utf-8") as cf:
self._config_file.parent.mkdir(parents=True, exist_ok=True)
config = self.get_config()
with self._config_file.open("w", encoding="utf-8") as cf:
json.dump(config, cf)

def _cleanup_config(self):
"""To prevent secrets leaking."""
if self.config_file.exists():
os.remove(self.config_file)
def _cleanup_config(self) -> None:
"""To prevent secrets leaking.
def _check_required_env_vars(self, env_vars):
If the config file exists, remove it.
"""
if self._config_file.exists():
Path(self._config_file).unlink()

def _check_required_env_vars(self, env_vars: Iterable[str]) -> None:
"""Check that required env vars are set."""
for env_var in env_vars:
assert os.environ.get(
env_var
), f"Environment variable '{env_var}' required for database type '{self.database}'."
try:
os.environ[env_var]
except KeyError as e:
raise MissingEnvVarError(env_var, self.database) from e

def _get_config_duckdb_sqlite(self):
def _get_config_duckdb_sqlite(self) -> dict:
"""Get config for duckdb or sqlite."""
self._check_required_env_vars(["EVIDENCE_CREDENTIALS_FILENAME"])
config = {
config: dict[str, t.Any] = {
"database": self.database,
"credentials": {"filename": os.environ["EVIDENCE_CREDENTIALS_FILENAME"]},
}
if self.database == "duckdb":
config["credentials"]["gitignoreDuckdb"] = os.environ.get(
"EVIDENCE_CREDENTIALS_GITIGNORE_DUCKDB"
"EVIDENCE_CREDENTIALS_GITIGNORE_DUCKDB",
)
return config

def _get_config_bigquery(self):
def _get_config_bigquery(self) -> dict:
"""Get config for BigQuery."""
self._check_required_env_vars(
[
"EVIDENCE_CREDENTIALS_CLIENT_EMAIL",
"EVIDENCE_CREDENTIALS_PRIVATE_KEY",
]
],
)
return {
"database": self.database,
Expand All @@ -115,3 +159,7 @@ def _get_config_bigquery(self):
"private_key": os.environ["EVIDENCE_CREDENTIALS_PRIVATE_KEY"],
},
}

def _get_config_mysql(self) -> dict:
"""Get config for MySQL."""
raise NotImplementedError
Loading

0 comments on commit cd44fd7

Please sign in to comment.