diff --git a/garak/generators/test.py b/garak/generators/test.py index 038060440..6617a76b9 100644 --- a/garak/generators/test.py +++ b/garak/generators/test.py @@ -5,6 +5,8 @@ from typing import List +import lorem + from garak.generators.base import Generator @@ -47,4 +49,17 @@ def _call_model(self, prompt: str, generations_this_call: int = 1) -> List[str]: ) -DEFAULT_CLASS = "Blank" +class Lipsum(Generator): + """Lorem Ipsum generator, so we can get non-zero outputs that vary""" + + supports_multiple_generations = False + generator_family_name = "Test" + name = "Lorem Ipsum" + + def _call_model( + self, prompt: str, generations_this_call: int = 1 + ) -> List[str | None]: + return [lorem.sentence() for i in range(generations_this_call)] + + +DEFAULT_CLASS = "Lipsum" diff --git a/pyproject.toml b/pyproject.toml index 4dbfc033a..b112d0a44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ dependencies = [ "fschat>=0.2.36", "litellm>=1.33.8", "jsonpath-ng>=1.6.1", + "lorem==0.1.1" ] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt index 4093acdf6..ac25f769e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,6 +28,7 @@ deepl==1.17.0 fschat>=0.2.36 litellm>=1.33.8 jsonpath-ng>=1.6.1 +lorem==0.1.1 # tests pytest>=8.0 requests-mock==1.12.1 diff --git a/tests/generators/test_test.py b/tests/generators/test_test.py new file mode 100644 index 000000000..918ba4a10 --- /dev/null +++ b/tests/generators/test_test.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +import garak._plugins +import garak.generators.base +import garak.generators.test + +TEST_GENERATORS = [ + a + for a, b in garak._plugins.enumerate_plugins("generators") + if b is True and a.startswith("generators.test") +] + + +@pytest.mark.parametrize("klassname", TEST_GENERATORS) +def test_test_instantiate(klassname): + g = garak._plugins.load_plugin(klassname) + assert isinstance(g, garak.generators.base.Generator) + + +@pytest.mark.parametrize("klassname", TEST_GENERATORS) +def test_test_gen(klassname): + g = garak._plugins.load_plugin(klassname) + for generations in (1, 50): + out = g.generate("", generations_this_call=generations) + assert isinstance(out, list), ".generate() must return a list" + assert ( + len(out) == generations + ), ".generate() must respect generations_per_call param" + for s in out: + assert ( + isinstance(s, str) or s is None + ), "generate()'s returned list's items must be string or None"