Additional fixes (#66)

README.md

@@ -87,6 +87,28 @@ When using mods that change resources in the game (or if Stellaris is updated in
 
 These values must be configured in the correct order, for vanilla Stellaris, this is the same order in which they are defined in the game file `common/strategic_resources/00_strategic_resources.txt`.
 
+### Names and Localizations
+
+Since Stellaris 3.4, the game no longer stores names (of countries, systems, ships, ...) in the save files as they appear in-game, but instead uses a templating system where names are stored in a more abstract representation. To make names show up in the same way as they do in-game, the dashboard program will try to load the information required from your game files. 
+
+If this system is not configured correctly, you will see some long and cryptic names like 
+```
+{"key": "format.gen_olig.1", "variables": [
+{"key": "generic_oli_desc", "value": {"key": "Sovereign"}}, 
+{"key": "generic_states", "value": {"key": "Realms"}}, 
+{"key": "This.GetSpeciesName", "value": {"key": "SPEC_Klenn"}}]}
+```
+when using the dashboard.
+
+If you are not using any mods and installed Stellaris in the default location at (`C:/Program Files (x86)/Steam/steamapps/common/Stellaris/` on windows), there is a good chance that everything will work without any additional actions. 
+
+However, if you use mods that add new names, if you play the game in another language, or if your Stellaris game files (or steam library) are in a different location, you will need to take some manual actions. While the dashboard application itself does not have any kind of localization (yet), you can also change the localization_file_dir to have names rendered in your preferred language.
+
+In this case, you can fix it like this:
+- Make sure there is a single folder containing all required localization files ending in `.yml`, including the files from your stellaris game and from all mods that introduce new names. You might need to collect these files yourself and copy them to a convenient location. The file names you use does not matter, the dashboard will take them all and build one big name mapping.
+- Update the `Stellaris localization folder` in the dashboard settings (or edit it in config.yml). Restart the dashboard program.
+- Look in the dashboard program for some output like `Loaded 162 localization files from <path-to-your-stellaris>/localisation/english` or similar that matches the number of files you prepared.
+
 ## How to improve performance
 
 If you find that the dashboard is too slow to browse, you can try some of these things:
@@ -100,7 +122,7 @@ If you find that the dashboard is too slow to browse, you can try some of these
 
 ### Multiplayer
 
-Support for Multiplayer is **experimental**. The dashboard will avoid showing information about other player controlled empires, even if the "Show all empires checkbox" is ticked in the settings. To use the dashboard in multiplayer, you must first configure your multiplayer username in the dashboard settings menu.
+Support for Multiplayer is **experimental**. The dashboard will avoid showing information about other player controlled empires, even if the "Show all empires" checkbox is ticked in the settings. To use the dashboard in multiplayer, you must first configure your multiplayer username in the dashboard settings menu.
 
 ### Observer Mode
 

stellarisdashboard/config.py

@@ -1,4 +1,5 @@
 import dataclasses
+import itertools
 import logging
 import multiprocessing as mp  # only to get the cpu count
 import pathlib
@@ -14,7 +15,7 @@
 
 LOG_FORMAT = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 
-CONFIG = None
+CONFIG: "Config" = None
 logger: logging.Logger = None
 
 
@@ -52,16 +53,15 @@ def _get_default_save_path():
         return home / "Documents/Paradox Interactive/Stellaris/save games/"
 
 
-def _get_default_localization_files() -> List[pathlib.Path]:
-    files = []
+def _get_default_localization_file_dir() -> pathlib.Path:
     for p in [
         pathlib.Path("C:/Program Files (x86)/Steam/steamapps/common/Stellaris/"),
         (pathlib.Path.home() / ".steam/steamapps/common/Stellaris/").absolute(),
     ]:
         p_abs = p / "localisation/english/"
-        files += p_abs.glob("*.yaml")
-        files += p_abs.glob("*.yml")
-    return files
+        if p_abs.exists():
+            return p_abs
+    return pathlib.Path(__file__).parent
 
 
 def _get_default_base_output_path():
@@ -178,7 +178,7 @@ def _get_default_base_output_path():
     mp_username="",
     base_output_path=_get_default_base_output_path(),
     threads=_get_default_thread_count(),
-    localization_files=_get_default_localization_files(),
+    localization_file_dir=_get_default_localization_file_dir(),
     port=28053,
     polling_interval=0.5,
     check_version=True,
@@ -205,7 +205,7 @@ class Config:
 
     save_file_path: pathlib.Path = None
     mp_username: str = None
-    localization_files: List[pathlib.Path] = dataclasses.field(default_factory=list)
+    localization_file_dir: pathlib.Path = None
     base_output_path: pathlib.Path = None
     threads: int = None
 
@@ -236,6 +236,7 @@ class Config:
     PATH_KEYS = {
         "base_output_path",
         "save_file_path",
+        "localization_file_dir",
     }
     BOOL_KEYS = {
         "check_version",
@@ -264,7 +265,7 @@ class Config:
     DICT_KEYS = {
         "tab_layout",
     }
-    LIST_KEYS = {"market_resources", "market_fee", "localization_files"}
+    LIST_KEYS = {"market_resources", "market_fee"}
     ALL_KEYS = (
         PATH_KEYS | BOOL_KEYS | INT_KEYS | FLOAT_KEYS | STR_KEYS | DICT_KEYS | LIST_KEYS
     )
@@ -278,8 +279,6 @@ def apply_dict(self, settings_dict):
                 logger.info(f"Ignoring unknown setting {key} with value {val}.")
                 continue
             old_val = self.__dict__.get(key)
-            if key == "localization_files":
-                val = [pathlib.Path(p) for p in val]
             if key in Config.BOOL_KEYS:
                 val = self._preprocess_bool(val)
             if key in Config.PATH_KEYS:
@@ -360,8 +359,6 @@ def get_dict(self):
                 if key in Config.PATH_KEYS:
                     val = str(val)
                 result[key] = val
-            if key == "localization_files":
-                result[key] = [str(p) for p in val]
         return result
 
     def get_adjustable_settings_dict(self):
@@ -396,6 +393,19 @@ def db_path(self) -> pathlib.Path:
             path.mkdir(parents=True)
         return path
 
+    @property
+    def localization_files(self):
+        files = list(
+            itertools.chain(
+                self.localization_file_dir.glob("*.yaml"),
+                self.localization_file_dir.glob("*.yml"),
+            )
+        )
+        logger.info(
+            f"Loaded {len(files)} localization files from {self.localization_file_dir}"
+        )
+        return files
+
 
 def _apply_existing_settings(config: Config):
     settings = dict(DEFAULT_SETTINGS)
@@ -427,6 +437,7 @@ def initialize():
         (CONFIG.base_output_path / "output").mkdir(parents=True)
 
     configure_logger()
+    CONFIG.write_to_file()
 
 
 def configure_logger():

stellarisdashboard/dashboard_app/settings.py

@@ -32,6 +32,12 @@ def _bool_to_lowercase(py_bool: bool) -> str:
             "name": "Save file path (applies after restart, set to empty to restore default, applies after restart)",
             "description": "Where the dashboard will look for new Stellaris save files.",
         },
+        "localization_file_dir": {
+            "type": t_str,
+            "value": current_values["localization_file_dir"],
+            "name": "Stellaris localization folder",
+            "description": "Path where the dashboard will look for Stellaris localization files that define how names are created. Applies after dashboard restart. See README.md for details.",
+        },
         "mp_username": {
             "type": t_str,
             "value": current_values["mp_username"],

stellarisdashboard/dashboard_app/utils.py

@@ -7,7 +7,7 @@
 
 logger = logging.getLogger(__name__)
 
-VERSION = "v3.0"
+VERSION = "v4.0"
 
 
 def parse_version(version: str):

stellarisdashboard/dashboard_app/visualization_data.py

@@ -723,7 +723,7 @@ def _iterate_popstats(
 
     def _get_key_from_popstats(self, ps: PopStatsType) -> str:
         assert isinstance(ps, datamodel.PopStatsByFaction)
-        return ps.faction.faction_name
+        return ps.faction.rendered_name
 
 
 class FactionDistributionDataContainer(AbstractPopStatsByFactionDataContainer):

stellarisdashboard/game_info.py

@@ -20,7 +20,7 @@ def load_name_mapping(self):
         self.name_mapping = {"global_event_country": "Global event country"}
         for p in self.localization_files:
             # manually parse yaml, yaml.safe_load doesnt seem to work
-            with open(p, "rt") as f:
+            with open(p, "rt", encoding="utf-8") as f:
                 for line in f:
                     try:
                         key, val, *rest = line.strip().split('"')

test/localization_test_files/english/name_list_HUMAN1_l_english.yml

@@ -1,4 +1,3 @@
 l_english:
- HUMAN1_SHIP_SpecialSymbols:0 "Special-Symbols. ''!§$%&/()"
- HUMAN1_SHIP_Umlaut:0 "ÄÖÜäöüéèê"
+ HUMAN1_SHIP_SpecialSymbols:0 "Special-Symbols. ''!$%&/()"
  HUM2_xxPUREBLOODS_ORD:0 "$O$ Purebloods"

test/names_test.py

@@ -146,16 +146,11 @@ def test_name_rendering_with_game_files(test_case: NameTestcase):
             expected="HUM2_KEY_DOES_NOT_EXIST",
             description="army, unknown returns top-level key",
         ),
-        # NameTestcase(
-        #     dict(key="HUMAN1_SHIP_Umlaut"),
-        #     expected="ÄÖÜäöüéèê",
-        #     description="ship, umlaut",
-        # ),
-        # NameTestcase(
-        #     dict(key="HUMAN1_SHIP_SpecialSymbols"),
-        #     expected=r"Special-Symbols. ''!§$%&/()",
-        #     description="ship, two words",
-        # ),
+        NameTestcase(
+            dict(key="HUMAN1_SHIP_SpecialSymbols"),
+            expected=r"Special-Symbols. ''!$%&/()",
+            description="ship, two words",
+        ),
         NameTestcase(
             dict(
                 key="format.gen_imp.1",