Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-38339: Update to latest Safir, use Settings for config #213

Merged
merged 7 commits into from
Mar 16, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Convert configuration to Settings
Use the Pydantic Settings class for configuration instead of a
dataclass with os.getenv initializers. Fix the typing of URLs to
validate syntax, which unfortunately requires allowing None for
the test suite and adding more verification elsewhere. Be a bit
more robust in URL construction to allow for trailing slashes in
the configured URLs.

Switching the type to HttpUrl requires doing some weird things to
override the defaults in fixtures, but this is apparently the
correct Pydantic way of doing it.
  • Loading branch information
rra committed Mar 16, 2023
commit 6e66965b8f7a519ad0f4e78ab42ef17d53918475
4 changes: 3 additions & 1 deletion src/mobu/business/tapqueryrunner.py
Original file line number Diff line number Diff line change
@@ -53,7 +53,9 @@ def __init__(

@staticmethod
def _make_client(token: str) -> pyvo.dal.TAPService:
tap_url = config.environment_url + "/api/tap"
if not config.environment_url:
raise RuntimeError("environment_url not set")
tap_url = str(config.environment_url).rstrip("/") + "/api/tap"

s = requests.Session()
s.headers["Authorization"] = "Bearer " + token
6 changes: 4 additions & 2 deletions src/mobu/cachemachine.py
Original file line number Diff line number Diff line change
@@ -24,10 +24,12 @@ def __init__(
self._session = session
self._token = token
self._username = username
if not config.environment_url:
raise RuntimeError("environment_url not set")
self._url = (
config.environment_url
str(config.environment_url).rstrip("/")
+ "/cachemachine/jupyter/"
+ config.cachemachine_image_policy
+ config.cachemachine_image_policy.value
)

async def get_latest_weekly(self) -> JupyterImage:
143 changes: 81 additions & 62 deletions src/mobu/config.py
Original file line number Diff line number Diff line change
@@ -2,83 +2,102 @@

from __future__ import annotations

import os
from dataclasses import dataclass
from enum import Enum
from pathlib import Path

__all__ = ["Configuration", "config"]
from pydantic import BaseSettings, Field, HttpUrl
from safir.logging import LogLevel

__all__ = [
"CachemachinePolicy",
"Configuration",
"config",
]

@dataclass
class Configuration:
"""Configuration for mobu."""

alert_hook: str | None = os.getenv("ALERT_HOOK")
"""The slack webhook used for alerting exceptions to slack.

Set with the ``ALERT_HOOK`` environment variable.
This is an https URL which should be considered secret.
If not set or set to "None", this feature will be disabled.
"""

autostart: str | None = os.getenv("AUTOSTART")
"""The path to a YAML file defining what flocks to automatically start.
class CachemachinePolicy(Enum):
"""Policy for what eligible images to retrieve from cachemachine."""

The YAML file should, if given, be a list of flock specifications. All
flocks specified there will be automatically started when mobu starts.
"""
available = "available"
desired = "desired"

environment_url: str = os.getenv("ENVIRONMENT_URL", "")
"""The URL of the environment to run tests against.

This is used for creating URLs to services, such as JupyterHub.
mobu will not work if this is not set.

Set with the ``ENVIRONMENT_URL`` environment variable.
"""
class Configuration(BaseSettings):
"""Configuration for mobu."""

cachemachine_image_policy: str = os.getenv(
"CACHEMACHINE_IMAGE_POLICY", "available"
alert_hook: HttpUrl | None = Field(
None,
title="Slack webhook URL used for sending alerts",
description=(
"An https URL, which should be considered secret. If not set or"
" set to `None`, this feature will be disabled."
),
env="ALERT_HOOK",
example="https://slack.example.com/ADFAW1452DAF41/",
)
"""Whether to use the images available on all nodes, or the images
desired by cachemachine. In instances where image streaming is enabled,
and therefore pulls are fast, ``desired`` is preferred. The default is
``available``.

Set with the ``CACHEMACHINE_IMAGE_POLICY`` environment variable.
"""

gafaelfawr_token: str | None = os.getenv("GAFAELFAWR_TOKEN")
"""The Gafaelfawr admin token to use to create user tokens.

This token is used to make an admin API call to Gafaelfawr to get a token
for the user. mobu will not work if this is not set.

Set with the ``GAFAELFAWR_TOKEN`` environment variable.
"""

name: str = os.getenv("SAFIR_NAME", "mobu")
"""The application's name, which doubles as the root HTTP endpoint path.

Set with the ``SAFIR_NAME`` environment variable.
"""

profile: str = os.getenv("SAFIR_PROFILE", "development")
"""Application run profile: "development" or "production".
autostart: Path | None = Field(
None,
title="Path to YAML file defining flocks to automatically start",
description=(
"If given, the YAML file must contain a list of flock"
" specifications. All flocks given there will be automatically"
" started when mobu starts."
),
env="AUTOSTART",
example="/etc/mobu/autostart.yaml",
)

Set with the ``SAFIR_PROFILE`` environment variable.
"""
environment_url: HttpUrl | None = Field(
None,
title="Base URL of the Science Platform environment",
description=(
"Used to create URLs to other services, such as Gafaelfawr and"
" JupyterHub. This is only optional to make writing the test"
" suite easier. If it is not set to a valid URL, mobu will abort"
" during startup."
),
env="ENVIRONMENT_URL",
example="https://data.example.org/",
)

logger_name: str = os.getenv("SAFIR_LOGGER", "mobu")
"""The root name of the application's logger.
cachemachine_image_policy: CachemachinePolicy = Field(
CachemachinePolicy.available,
field="Class of cachemachine images to use",
description=(
"Whether to use the images available on all nodes, or the images"
" desired by cachemachine. In instances where image streaming is"
" enabled and therefore pulls are fast, ``desired`` is preferred."
" The default is ``available``."
),
env="CACHEMACHINE_IMAGE_POLICY",
example=CachemachinePolicy.desired,
)

Set with the ``SAFIR_LOGGER`` environment variable.
"""
gafaelfawr_token: str | None = Field(
None,
field="Gafaelfawr admin token used to create user tokens",
description=(
"This token is used to make an admin API call to Gafaelfawr to"
" get a token for the user. This is only optional to make writing"
" tests easier. mobu will abort during startup if it is not set."
),
env="GAFAELFAWR_TOKEN",
example="gt-vilSCi1ifK_MyuaQgMD2dQ.d6SIJhowv5Hs3GvujOyUig",
)

log_level: str = os.getenv("SAFIR_LOG_LEVEL", "INFO")
"""The log level of the application's logger.
name: str = Field(
"mobu",
title="Name of application",
description="Doubles as the root HTTP endpoint path.",
env="SAFIR_NAME",
)

Set with the ``SAFIR_LOG_LEVEL`` environment variable.
"""
log_level: LogLevel = Field(
LogLevel.INFO,
title="Log level of the application's logger",
env="SAFIR_LOG_LEVEL",
)


config = Configuration()
6 changes: 5 additions & 1 deletion src/mobu/jupyterclient.py
Original file line number Diff line number Diff line change
@@ -222,7 +222,11 @@ def __init__(
self.user = user
self.log = log
self.config = jupyter_config
self.jupyter_url = config.environment_url + jupyter_config.url_prefix
if not config.environment_url:
raise RuntimeError("environment_url not set")
self.jupyter_url = (
str(config.environment_url).rstrip("/") + jupyter_config.url_prefix
)

xsrftoken = "".join(
random.choices(string.ascii_uppercase + string.digits, k=16)
2 changes: 2 additions & 0 deletions src/mobu/main.py
Original file line number Diff line number Diff line change
@@ -55,6 +55,8 @@
async def startup_event() -> None:
if not config.environment_url:
raise RuntimeError("ENVIRONMENT_URL was not set")
if not config.gafaelfawr_token:
raise RuntimeError("GAFAELFAWR_TOKEN was not set")
await monkey_business_manager.init()
if config.autostart:
await autostart()
6 changes: 5 additions & 1 deletion src/mobu/models/user.py
Original file line number Diff line number Diff line change
@@ -95,7 +95,11 @@ class AuthenticatedUser(User):
async def create(
cls, user: User, scopes: list[str], session: ClientSession
) -> Self:
token_url = f"{config.environment_url}/auth/api/v1/tokens"
if not config.environment_url:
raise RuntimeError("environment_url not set")
token_url = (
str(config.environment_url).rstrip("/") + "/auth/api/v1/tokens"
)
data: dict[str, Any] = {
"username": user.username,
"name": "Mobu Test User",
5 changes: 2 additions & 3 deletions tests/autostart_test.py
Original file line number Diff line number Diff line change
@@ -46,9 +46,8 @@ def configure_autostart(
) -> Iterator[None]:
"""Set up the autostart configuration."""
mock_gafaelfawr(mock_aioresponses, any_uid=True)
autostart_path = tmp_path / "autostart.yaml"
autostart_path.write_text(AUTOSTART_CONFIG)
config.autostart = str(autostart_path)
config.autostart = tmp_path / "autostart.yaml"
config.autostart.write_text(AUTOSTART_CONFIG)
yield
config.autostart = None

7 changes: 5 additions & 2 deletions tests/business/jupyterloginloop_test.py
Original file line number Diff line number Diff line change
@@ -172,7 +172,8 @@ async def test_alert(
assert data["business"]["failure_count"] > 0

# Check that an appropriate error was posted.
url = urljoin(config.environment_url, "/nb/hub/spawn")
assert config.environment_url
url = urljoin(str(config.environment_url), "/nb/hub/spawn")
assert slack.messages == [
{
"blocks": [
@@ -246,8 +247,10 @@ async def test_redirect_loop(
assert data["business"]["failure_count"] > 0

# Check that an appropriate error was posted.
assert config.environment_url
url = urljoin(
config.environment_url, "/nb/hub/api/users/testuser1/server/progress"
str(config.environment_url),
"/nb/hub/api/users/testuser1/server/progress",
)
assert slack.messages == [
{
10 changes: 6 additions & 4 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -14,6 +14,8 @@
from asgi_lifespan import LifespanManager
from fastapi import FastAPI
from httpx import AsyncClient
from pydantic import HttpUrl
from pydantic.tools import parse_obj_as
from safir.testing.slack import MockSlackWebhook, mock_slack_webhook

from mobu import main
@@ -39,10 +41,10 @@ def configure() -> Iterator[None]:
minimal test configuration and a unique admin token that is replaced after
the test runs.
"""
config.environment_url = "https://test.example.com"
config.environment_url = parse_obj_as(HttpUrl, "https://test.example.com")
config.gafaelfawr_token = make_gafaelfawr_token()
yield
config.environment_url = ""
config.environment_url = None
config.gafaelfawr_token = None


@@ -107,6 +109,6 @@ async def mock_ws_connect(url: str, **kwargs: Any) -> MockJupyterWebSocket:

@pytest.fixture
def slack(respx_mock: respx.Router) -> Iterator[MockSlackWebhook]:
config.alert_hook = "https://slack.example.com/services/XXXX/YYYYY"
yield mock_slack_webhook(config.alert_hook, respx_mock)
config.alert_hook = parse_obj_as(HttpUrl, "https://slack.example.com/XXXX")
yield mock_slack_webhook(str(config.alert_hook), respx_mock)
config.alert_hook = None
7 changes: 5 additions & 2 deletions tests/monkeyflocker_test.py
Original file line number Diff line number Diff line change
@@ -22,6 +22,8 @@
from mobu.config import config
from monkeyflocker.cli import main

from .support.gafaelfawr import make_gafaelfawr_token

APP_SOURCE = """
from collections.abc import Awaitable, Callable

@@ -31,7 +33,7 @@

from mobu.config import config
from mobu.main import app
from tests.support.gafaelfawr import make_gafaelfawr_token, mock_gafaelfawr
from tests.support.gafaelfawr import mock_gafaelfawr
from tests.support.jupyter import mock_jupyter


@@ -58,7 +60,6 @@ async def dispatch(

@app.on_event("startup")
async def startup_event() -> None:
config.gafaelfawr_token = make_gafaelfawr_token()
mocked = aioresponses()
mocked.start()
mock_gafaelfawr(mocked)
@@ -109,13 +110,15 @@ def app_url(tmp_path: Path) -> Iterator[str]:

cmd = ["uvicorn", "--fd", "0", "testing:app"]
logging.info("Starting server with command %s", " ".join(cmd))
assert config.environment_url
p = subprocess.Popen(
cmd,
cwd=str(tmp_path),
stdin=s.fileno(),
env={
**os.environ,
"ENVIRONMENT_URL": config.environment_url,
"GAFAELFAWR_TOKEN": make_gafaelfawr_token(),
"PYTHONPATH": os.getcwd(),
},
)
2 changes: 1 addition & 1 deletion tests/support/gafaelfawr.py
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@ def handler(url: str, **kwargs: Any) -> CallbackResult:
response = {"token": make_gafaelfawr_token(kwargs["json"]["username"])}
return CallbackResult(payload=response, status=200)

base_url = config.environment_url
base_url = str(config.environment_url).rstrip("/")
mocked.post(
f"{base_url}/auth/api/v1/tokens", callback=handler, repeat=True
)
5 changes: 3 additions & 2 deletions tests/support/jupyter.py
Original file line number Diff line number Diff line change
@@ -50,10 +50,11 @@ class JupyterState(Enum):

def _url(route: str, regex: bool = False) -> str | Pattern[str]:
"""Construct a URL for JupyterHub/Proxy."""
base_url = str(config.environment_url).rstrip("/")
if not regex:
return f"{config.environment_url}/nb/{route}"
return f"{base_url}/nb/{route}"

prefix = re.escape(f"{config.environment_url}/nb/")
prefix = re.escape(f"{base_url}/nb/")
return re.compile(prefix + route)