Merge pull request #164 from thom311/th/output-base-option

[th/output-base-option] add `--output-base` option to pre-determine the name of the result files

#164

evaluator.py

@@ -164,13 +164,20 @@ def parse_args() -> argparse.Namespace:
         description="Tool to evaluate TFT Flow test results"
     )
     parser.add_argument(
-        "config", metavar="config", type=str, help="Yaml file with tft test threshholds"
+        "config",
+        metavar="config",
+        type=str,
+        help='YAML configuration file with tft test threshholds. See "eval-config.yaml".',
     )
     parser.add_argument(
-        "logs", type=str, help="Directory containing TFT log files to evaluate"
+        "logs",
+        type=str,
+        help='Result file from ocp-traffic-flow-tests. This defaults to "./ft-logs/$TIMESTAMP.json".',
     )
     parser.add_argument(
-        "output", type=str, help="Output file to write evaluation results to"
+        "output",
+        type=str,
+        help="Output file to write evaluation results to.",
     )
     common.log_argparse_add_argument_verbose(parser)
 

main.py

@@ -20,15 +20,30 @@ def parse_args() -> argparse.Namespace:
         "config",
         metavar="config",
         type=str,
-        help="Yaml file with test configuration (see config.yaml)",
+        help="YAML file with test configuration (see config.yaml)",
     )
     parser.add_argument(
         "evaluator_config",
         nargs="?",
         metavar="evaluator_config",
         type=str,
-        help="Yaml file with configuration for scoring test results (see eval-config.yaml)",
+        help="YAML file with configuration for scoring test results (see eval-config.yaml). "
+        "This parameter is optional. If provided, this effectively runs "
+        "./evaluator.py ${evaluator_config} ${test_result} ${evaluator_result}` on "
+        "the result file. You can run this step separately.",
     )
+    parser.add_argument(
+        "-o",
+        "--output-base",
+        type=str,
+        default=None,
+        help="The base name for the result files. If specified, the result will be "
+        'written to "${output_base}$(printf \'%%03d\' "$number").json" where ${number} is the '
+        "zero-based index of the test. This can include the directory name and is relative to "
+        'the current directory. If unspecified, the files are written to "${logs}/${timestamp}.json" '
+        'where "${logs}" can be specified in the config file (and defaults to "./ft-logs/").',
+    )
+
     common.log_argparse_add_argument_verbosity(parser)
 
     args = parser.parse_args()
@@ -47,6 +62,7 @@ def main() -> None:
     tc = TestConfig(
         config_path=args.config,
         evaluator_config=args.evaluator_config,
+        output_base=args.output_base,
     )
     tft = TrafficFlowTests()
 

testConfig.py

@@ -1,5 +1,6 @@
 import abc
 import dataclasses
+import datetime
 import json
 import logging
 import os
@@ -461,6 +462,26 @@ def logs_abspath(self) -> pathlib.Path:
             cwd=self.config.test_config.cwddir,
         )
 
+    def get_output_file(self) -> pathlib.Path:
+        output_base = self.config.test_config.output_base
+
+        if output_base is None:
+            timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+            return self.logs_abspath / f"{timestamp}.json"
+
+        path = common.path_norm(
+            output_base,
+            cwd=self.config.test_config.cwddir,
+            preserve_dir=True,
+        )
+        tft_idx = f"{self.config.tft_idx:03d}"
+        if path[-1] == "/":
+            base = "result-"
+        else:
+            base = os.path.basename(path)
+            path = os.path.dirname(path)
+        return pathlib.Path(os.path.join(path, f"{base}{tft_idx}.json"))
+
 
 @strict_dataclass
 @dataclass(frozen=True, kw_only=True)
@@ -477,6 +498,10 @@ def __post_init__(self) -> None:
     def test_config(self) -> "TestConfig":
         return self._owner_reference.get(TestConfig)
 
+    @property
+    def tft_idx(self) -> int:
+        return self.yamlidx
+
     def serialize(self) -> dict[str, Any]:
         return {
             "tft": [c.serialize() for c in self.tft],
@@ -536,6 +561,7 @@ class TestConfig:
     _client_infra: Optional[K8sClient]
     _client_lock: threading.Lock
     evaluator_config: Optional[str]
+    output_base: Optional[str]
 
     @property
     def mode(self) -> ClusterMode:
@@ -575,6 +601,7 @@ def __init__(
         config_path: Optional[str] = None,
         kubeconfigs: Optional[tuple[str, Optional[str]]] = None,
         evaluator_config: Optional[str] = None,
+        output_base: Optional[str] = None,
         cwddir: str = ".",
     ) -> None:
 
@@ -624,6 +651,10 @@ def __init__(
         self._client_tenant = None
         self._client_infra = None
 
+        if not output_base:
+            output_base = None
+        self.output_base = output_base
+
         kubeconfigs_cwd = cwddir
         if kubeconfigs is not None:
             kubeconfigs_pair = kubeconfigs

tests/test_testConfig.py

@@ -1,5 +1,6 @@
 import json
 import os
+import pathlib
 import sys
 import yaml
 
@@ -170,7 +171,11 @@ def test_config2() -> None:
 kubeconfig_infra: /path/to/kubeconfig_infra
 """
     )
-    tc = testConfig.TestConfig(full_config=full_config, kubeconfigs=None)
+    tc = testConfig.TestConfig(
+        full_config=full_config,
+        kubeconfigs=None,
+        output_base="/tmp/",
+    )
     assert isinstance(tc, testConfig.TestConfig)
 
     assert tc.config.kubeconfig == "/path/to/kubeconfig"
@@ -196,6 +201,7 @@ def test_config2() -> None:
     assert tc.config.tft[0].connections[1].test_type == TestType.SIMPLE
     assert tc.config.tft[0].connections[1].client[0].args == ("foo", "-x x")
     assert tc.config.tft[0].connections[1].server[0].args == ("hi x",)
+    assert tc.config.tft[0].get_output_file() == pathlib.Path("/tmp/result-000.json")
 
     _check_testConfig(tc)
 
@@ -215,7 +221,9 @@ def test_config2() -> None:
 """
     )
     tc = testConfig.TestConfig(
-        full_config=full_config, kubeconfigs=testConfigKubeconfigsArgs1
+        full_config=full_config,
+        kubeconfigs=testConfigKubeconfigsArgs1,
+        output_base="/tmp/result2-",
     )
     assert isinstance(tc, testConfig.TestConfig)
 
@@ -230,5 +238,6 @@ def test_config2() -> None:
         common.enum_convert_list(TestCaseType, "*")
     )
     assert tc.config.tft[0].connections[0].name == "Connection Test 1/1"
+    assert tc.config.tft[0].get_output_file() == pathlib.Path("/tmp/result2-000.json")
 
     _check_testConfig(tc)

trafficFlowTests.py

@@ -1,4 +1,3 @@
-import datetime
 import json
 import logging
 import task
@@ -56,10 +55,8 @@ def _cleanup_previous_testspace(self, cfg_descr: ConfigDescriptor) -> None:
         )
 
     def _create_log_paths_from_tests(self, test: testConfig.ConfTest) -> Path:
-        log_path = test.logs_abspath
-        log_path.mkdir(parents=True, exist_ok=True)
-        timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
-        log_file = log_path / f"{timestamp}.json"
+        log_file = test.get_output_file()
+        log_file.parent.mkdir(parents=True, exist_ok=True)
         logger.info(f"Logs will be written to {log_file}")
         return log_file