Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gather BIT metrics [implementation] #3122

Merged
merged 20 commits into from
Jul 11, 2024
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
ad5fe37
Inflate 3116
originalsouth Jun 20, 2024
aa70bf7
Set default off
originalsouth Jun 20, 2024
33139a6
Add yielded oois
originalsouth Jun 20, 2024
79a94bf
Prevent bytes-strings
originalsouth Jun 20, 2024
5d50bfe
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jun 24, 2024
54cea93
Add bit metric "parser"
originalsouth Jun 24, 2024
90fbda2
Manual merge of origin/main and address comments by @ammer92
originalsouth Jul 4, 2024
030ec3d
Remove annotations from settings
originalsouth Jul 5, 2024
059c461
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jul 5, 2024
8216ab6
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jul 8, 2024
d5c3d1b
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jul 9, 2024
2bc2ead
Merge branch 'main' into feature/3116
underdarknl Jul 9, 2024
7ee2621
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jul 9, 2024
aa74603
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jul 10, 2024
2912965
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jul 10, 2024
f70c5c3
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jul 10, 2024
2430f64
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jul 10, 2024
2ab00ca
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jul 11, 2024
89c9806
Make analyze-bit-metric.py more user friendly
originalsouth Jul 11, 2024
6dc35e5
Merge remote-tracking branch 'origin/main' into feature/3116
originalsouth Jul 11, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions octopoes/octopoes/config/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,3 +89,4 @@ def settings_customise_sources(
DEFAULT_LIMIT = 50
DEFAULT_OFFSET = 0
QUEUE_NAME_OCTOPOES: str = "octopoes"
GATHER_BIT_METRICS: bool = False
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure what the intention is, but usually feature flags are configurable. Perhaps this fits better in the Settings class and documented. This could work if it's only a private debug variable for development, although the location is bit misleading since it's placed next to the constants.

The bool hint is unnecessary here

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The intention is to have a developer flag to toggle gathering [bit] metrics (without exposing it to the public). Please let me know what the best way of defining this would be.

11 changes: 7 additions & 4 deletions octopoes/octopoes/core/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from amqp import AMQPError

from octopoes.config.settings import QUEUE_NAME_OCTOPOES, Settings
from octopoes.config.settings import GATHER_BIT_METRICS, QUEUE_NAME_OCTOPOES, Settings
from octopoes.core.service import OctopoesService
from octopoes.events.manager import EventManager, get_rabbit_channel
from octopoes.repositories.ooi_repository import XTDBOOIRepository
Expand Down Expand Up @@ -39,6 +39,9 @@ def bootstrap_octopoes(settings: Settings, client: str, xtdb_session: XTDBSessio
origin_param_repository = XTDBOriginParameterRepository(event_manager, xtdb_session)
scan_profile_repository = XTDBScanProfileRepository(event_manager, xtdb_session)

octopoes = OctopoesService(ooi_repository, origin_repository, origin_param_repository, scan_profile_repository)

return octopoes
if GATHER_BIT_METRICS:
return OctopoesService(
ooi_repository, origin_repository, origin_param_repository, scan_profile_repository, xtdb_session
)
else:
return OctopoesService(ooi_repository, origin_repository, origin_param_repository, scan_profile_repository)
27 changes: 24 additions & 3 deletions octopoes/octopoes/core/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
from collections.abc import Callable, ValuesView
from datetime import datetime, timezone
from logging import getLogger
from time import perf_counter
from typing import overload

from bits.definitions import get_bit_definitions
from bits.runner import BitRunner
from pydantic import TypeAdapter

from octopoes.config.settings import (
DEFAULT_LIMIT,
Expand Down Expand Up @@ -42,6 +44,7 @@
from octopoes.repositories.origin_parameter_repository import OriginParameterRepository
from octopoes.repositories.origin_repository import OriginRepository
from octopoes.repositories.scan_profile_repository import ScanProfileRepository
from octopoes.xtdb.client import Operation, OperationType, XTDBSession

logger = getLogger(__name__)
settings = Settings()
Expand All @@ -66,11 +69,13 @@ def __init__(
origin_repository: OriginRepository,
origin_parameter_repository: OriginParameterRepository,
scan_profile_repository: ScanProfileRepository,
session: XTDBSession | None = None,
):
self.ooi_repository = ooi_repository
self.origin_repository = origin_repository
self.origin_parameter_repository = origin_parameter_repository
self.scan_profile_repository = scan_profile_repository
self.session = session

@overload
def _populate_scan_profiles(self, oois: ValuesView[OOI], valid_time: datetime) -> ValuesView[OOI]: ...
Expand Down Expand Up @@ -199,13 +204,29 @@ def _run_inference(self, origin: Origin, valid_time: datetime) -> None:
config = configs[-1].config

try:
resulting_oois = BitRunner(bit_definition).run(source, list(parameters.values()), config=config)
if isinstance(self.session, XTDBSession):
start = perf_counter()
resulting_oois = BitRunner(bit_definition).run(source, list(parameters.values()), config=config)
stop = perf_counter()
metrics: dict[str, str] = {
"bit": bit_definition.id,
"config": json.dumps(config),
"elapsed": str(stop - start),
"parameters": TypeAdapter(list[OOI]).dump_json(list(parameters.values())).decode(),
"source": source.model_dump_json(),
"xt/id": "BIT_METRIC",
"yield": TypeAdapter(list[OOI]).dump_json(resulting_oois).decode(),
}
ops: list[Operation] = [(OperationType.PUT, metrics, valid_time)]
self.session.client.submit_transaction(ops)
self.save_origin(origin, resulting_oois, valid_time)
underdarknl marked this conversation as resolved.
Show resolved Hide resolved
else:
resulting_oois = BitRunner(bit_definition).run(source, list(parameters.values()), config=config)
self.save_origin(origin, resulting_oois, valid_time)
except Exception as e:
logger.exception("Error running inference", exc_info=e)
return

self.save_origin(origin, resulting_oois, valid_time)

@staticmethod
def check_path_level(path_level: int | None, current_level: int):
return path_level is not None and path_level >= current_level
Expand Down
103 changes: 103 additions & 0 deletions octopoes/tools/analyze-bit-metric.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
#!/usr/bin/env python

import json
import logging
from datetime import datetime

import click
from xtdb_client import XTDBClient

logger = logging.getLogger(__name__)


class BitMetric:
def __init__(self, data):
date_format = "%Y-%m-%dT%H:%M:%SZ"
self.txTime: datetime = datetime.strptime(data["txTime"], date_format)
self.txId: int = int(data["txId"])
self.validTime: datetime = datetime.strptime(data["validTime"], date_format)
self.contentHash: str = data["contentHash"]
underdarknl marked this conversation as resolved.
Show resolved Hide resolved
self.yld: dict[str, str] = json.loads(data["doc"]["yield"])
self.cfg: dict[str, str] = json.loads(data["doc"]["config"])
self.src: dict[str, str] = json.loads(data["doc"]["source"])
self.name: str = data["doc"]["bit"]
self.pms: list[dict[str, str]] = json.loads(data["doc"]["parameters"])
self.elapsed: list[dict[str, str]] = json.loads(data["doc"]["elapsed"])

def empty(self):
return len(self.yld) == 0

def __eq__(self, val):
return all([getattr(self, op) == getattr(val, op) for op in ["src", "cfg", "yld", "name", "pms"]])
underdarknl marked this conversation as resolved.
Show resolved Hide resolved

def __hash__(self):
return hash(str(hash(self.txTime)) + self.contentHash)


@click.group(
context_settings={
"help_option_names": ["-h", "--help"],
"max_content_width": 120,
"show_default": True,
}
)
@click.option("-n", "--node", default="0", help="XTDB node")
@click.option(
"-u",
"--url",
default="http://localhost:3000",
help="XTDB server base url",
)
@click.option(
"-t",
"--timeout",
type=int,
default=5000,
help="XTDB request timeout (in ms)",
)
@click.option("-v", "--verbosity", count=True, help="Increase the verbosity level")
@click.pass_context
def cli(ctx: click.Context, url: str, node: str, timeout: int, verbosity: int):
verbosities = [logging.WARN, logging.INFO, logging.DEBUG]
try:
if verbosity:
logging.basicConfig(level=verbosities[verbosity - 1])
except IndexError:
raise click.UsageError("Invalid verbosity level (use -v, -vv, or -vvv)")

client = XTDBClient(url, node, timeout)
logger.info("Instantiated XTDB client with endpoint %s for node %s", url, node)

ctx.ensure_object(dict)
ctx.obj["client"] = client
ctx.obj["raw_bit_metrics"] = client.history("BIT_METRIC", False, True)
ctx.obj["bit_metrics"] = list(map(lambda x: BitMetric(x), ctx.obj["raw_bit_metrics"]))


@cli.command(help="Returns the raw bit metric")
@click.pass_context
def raw(ctx: click.Context):
click.echo(ctx.obj["raw_bit_metrics"])


@cli.command(help="Returns the parsed metric")
@click.pass_context
def parse(ctx: click.Context):
underdarknl marked this conversation as resolved.
Show resolved Hide resolved
metrics = {
"total": len(ctx.obj["bit_metrics"]),
"total_elapsed": sum([bm.elapsed for bm in ctx.obj["bit_metrics"]]),
"empty": len([bm for bm in ctx.obj["bit_metrics"] if bm.empty()]),
"empty_elapsed": sum([bm.elapsed for bm in ctx.obj["bit_metrics"] if bm.empty()]),
"useful": len([bm for bm in ctx.obj["bit_metrics"] if not bm.empty()]),
"useful_elapsed": sum([bm.elapsed for bm in ctx.obj["bit_metrics"] if not bm.empty()]),
"futile_runs": {
bm.name: ctx.obj["bit_metrics"].count(bm) - 1
for bm in ctx.obj["bit_metrics"]
if ctx.obj["bit_metrics"].count(bm) > 1
},
}
click.echo(json.dumps(metrics))


if __name__ == "__main__":
cli()