From 5d9f435214f89259a828660e28de55357085c30f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elias=20D=C3=B6hne?= Date: Sat, 7 May 2022 18:08:42 +0200 Subject: [PATCH] Market price graphs (#64) --- README.md | 28 ++++ configure_layout.md | 2 +- mod/stellaris_dashboard/descriptor.mod | 2 +- stellarisdashboard/config.py | 67 +++++++-- .../dashboard_app/graph_ledger.py | 9 +- stellarisdashboard/dashboard_app/utils.py | 2 +- .../dashboard_app/visualization_data.py | 138 +++++++++++++++++- stellarisdashboard/datamodel.py | 57 ++++++++ stellarisdashboard/parsing/timeline.py | 102 +++++++++++++ 9 files changed, 388 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 72b13cc..a984851 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,34 @@ For balance and immersion, only some information about other empires is shown by - `Store data of all countries`: This will read detailed budgets and pop statistics for non-player countries. It will increase the processing time and database file size, but will allow you to inspect other countries by selecting them from the dropdown menu at the top. (Basic economy information is always read, this setting only controls the very detailed budgets) - `Filter history ledger by event type`: By default, the event ledger does not show everything on the main page. For example, more detailed events like leader level-ups are only shown on that specific leader's ledger entry. This setting allows you to change this behavior and see all events on the main page. +## Market Price Graphs + +The dashboard includes graphs of market prices for each resource in the game. To get the correct values, you will need to manually configure some things in the file `config.yml`, as these settings are not available in the built-in settings page. + +If `config.yml` does not exist, go to the settings page linked at the top (or `localhost:28053/settings/`) and hit "Apply Settings". This will create the file with all of your current settings. + +These configurations are applied only when preparing the graph, so you can adjust them at any time in the configuration without reprocessing any data. + +### Market fees + +Currently, there is no easy way to get the market fee information from the save files. To still get the correct numbers in the graph, you can add the fee manually in the configuration by creating additional entries in the `market_fee` section. By default, a constant fee of 30% is assumed. + +For example, to configure a game where the market_fee changed to 20% in 2240 and 5% in 2300, you could change the market_fee section like this: +``` +market_fee: +- {date: 2200.01.01, fee: 0.3} +- {date: 2240.01.01, fee: 0.2} +- {date: 2300.01.01, fee: 0.05} +``` + +### Resources + +The default resource configuration should be correct for the current vanilla Stellaris game. + +When using mods that change resources in the game (or if Stellaris is updated in the future), you might need to manually adjust the resource configuration in `config.yml` to have the data collected for the additional resources. + +These values must be configured in the correct order, for vanilla Stellaris, this is the same order in which they are defined in the game file `common/strategic_resources/00_strategic_resources.txt`. + ## How to improve performance If you find that the dashboard is too slow to browse, you can try some of these things: diff --git a/configure_layout.md b/configure_layout.md index 4951f5c..98af374 100644 --- a/configure_layout.md +++ b/configure_layout.md @@ -2,7 +2,7 @@ Disabling some graphs means that the dashboard has to do less work, making it faster to browse. Also, you can remove any graphs that you don't care about and arrange them into tabs in whatever way you prefer. -The configuration is written in YAML format which is quite easy to understand. You can chose almost any name for the tabs, except for "Galaxy Map", which is reserved. +The configuration is written in YAML format which is quite easy to understand. You can chose almost any name for the tabs, except for "Galaxy Map" and "Markets", which are reserved for special tabs and will be ignored. For example this would be a valid configuration you can add to the config.yml: ```yaml diff --git a/mod/stellaris_dashboard/descriptor.mod b/mod/stellaris_dashboard/descriptor.mod index e13c164..16f25ec 100644 --- a/mod/stellaris_dashboard/descriptor.mod +++ b/mod/stellaris_dashboard/descriptor.mod @@ -1,5 +1,5 @@ name="Stellaris Dashboard" -version="v2.0-alpha" +version="v3.0" tags={ "Utilities" "Gameplay" diff --git a/stellarisdashboard/config.py b/stellarisdashboard/config.py index b8b3e9e..fc62b15 100644 --- a/stellarisdashboard/config.py +++ b/stellarisdashboard/config.py @@ -6,7 +6,7 @@ import sys import traceback from collections import defaultdict -from typing import List, Dict +from typing import List, Dict, Optional, Any import yaml @@ -16,7 +16,7 @@ LOG_FORMAT = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") CONFIG = None -logger = None +logger: logging.Logger = None def initialize_logger(): @@ -57,6 +57,8 @@ def _get_default_base_output_path(): return pathlib.Path.cwd() / "output" +GALAXY_MAP_TAB = "Galaxy Map" +MARKET_TAB = "Markets" DEFAULT_TAB_LAYOUT = { "Budget": [ "energy_budget", @@ -130,8 +132,35 @@ def _get_default_base_output_path(): "victory_score_graph", "victory_economy_score_graph", ], + MARKET_TAB: [], # filled dynamically based on resource config } -GALAXY_MAP_TAB = "Galaxy Map" +DEFAULT_MARKET_RESOURCES = [ + # all available resources, in the order in which they are defined in the game files + # common/strategic_resources/00_strategic_resources.txt + # Put None as price of non-tradeable resources (These must still be added because ordering matters) + {"name": "time", "base_price": None}, + {"name": "energy", "base_price": None}, + {"name": "minerals", "base_price": 1}, + {"name": "food", "base_price": 1}, + {"name": "physics_research", "base_price": None}, + {"name": "society_research", "base_price": None}, + {"name": "engineering_research", "base_price": None}, + {"name": "influence", "base_price": None}, + {"name": "unity", "base_price": None}, + {"name": "consumer_goods", "base_price": 2}, + {"name": "alloys", "base_price": 4}, + {"name": "volatile_motes", "base_price": 10}, + {"name": "exotic_gases", "base_price": 10}, + {"name": "rare_crystals", "base_price": 10}, + {"name": "sr_living_metal", "base_price": 20}, + {"name": "sr_zro", "base_price": 20}, + {"name": "sr_dark_matter", "base_price": 20}, + {"name": "nanites", "base_price": None}, + {"name": "minor_artifacts", "base_price": None}, + {"name": "menace", "base_price": None}, +] +DEFAULT_MARKET_FEE = [{"date": "2200.01.01", "fee": 0.3}] + DEFAULT_SETTINGS = dict( save_file_path=_get_default_save_path(), @@ -153,6 +182,8 @@ def _get_default_base_output_path(): plot_width=1150, plot_height=640, tab_layout=DEFAULT_TAB_LAYOUT, + market_resources=DEFAULT_MARKET_RESOURCES, + market_fee=DEFAULT_MARKET_FEE, ) @@ -186,6 +217,8 @@ class Config: debug_mode: bool = False tab_layout: Dict[str, List[str]] = None + market_resources: List[Dict[str, Any]] = None + market_fee: List[Dict[str, float]] = None PATH_KEYS = { "base_output_path", @@ -215,11 +248,21 @@ class Config: "save_name_filter", "log_level", } - DICT_KEYS = {"tab_layout"} - ALL_KEYS = PATH_KEYS | BOOL_KEYS | INT_KEYS | FLOAT_KEYS | STR_KEYS | DICT_KEYS + DICT_KEYS = { + "tab_layout", + } + LIST_KEYS = { + "market_resources", + "market_fee", + } + ALL_KEYS = ( + PATH_KEYS | BOOL_KEYS | INT_KEYS | FLOAT_KEYS | STR_KEYS | DICT_KEYS | LIST_KEYS + ) def apply_dict(self, settings_dict): logger.info("Updating settings") + tab_layout = self._preprocess_tab_layout(settings_dict) + settings_dict["tab_layout"] = tab_layout for key, val in settings_dict.items(): if key not in Config.ALL_KEYS: logger.info(f"Ignoring unknown setting {key} with value {val}.") @@ -231,8 +274,7 @@ def apply_dict(self, settings_dict): val = self._process_path_keys(key, val) if val is None: continue - if key == "tab_layout": - val = self._process_tab_layout(val) + self.__setattr__(key, val) if val != old_val: logger.info( @@ -263,7 +305,8 @@ def _process_path_keys(self, key, val): return return val - def _process_tab_layout(self, layout_dict): + def _preprocess_tab_layout(self, settings_dict): + layout_dict = settings_dict.get("tab_layout", DEFAULT_TAB_LAYOUT) if not isinstance(layout_dict, dict): logger.error(f"Invalid tab layout configuration: {layout_dict}") logger.info(f"Falling back to default tab layout.") @@ -272,7 +315,13 @@ def _process_tab_layout(self, layout_dict): for tab, plot_list in layout_dict.items(): if tab == GALAXY_MAP_TAB: logger.warning(f"Ignoring tab {tab}, it is reserved for the galaxy map") - pass + continue + if tab == MARKET_TAB: + logger.warning( + f"Ignoring values for tab {tab}, it is filled dynamically" + ) + processed[tab] = [] + continue if not isinstance(plot_list, list): logger.warning(f"Ignoring invalid graph list for tab {tab}") pass diff --git a/stellarisdashboard/dashboard_app/graph_ledger.py b/stellarisdashboard/dashboard_app/graph_ledger.py index 5999f13..dc69c7a 100644 --- a/stellarisdashboard/dashboard_app/graph_ledger.py +++ b/stellarisdashboard/dashboard_app/graph_ledger.py @@ -257,9 +257,12 @@ def update_content( children = [] if tab_value in config.CONFIG.tab_layout: - plots = visualization_data.get_plot_specifications_for_tab_layout().get( - tab_value - ) + if tab_value == config.MARKET_TAB: + plots = visualization_data.get_market_graphs(config.CONFIG.market_resources) + else: + plots = visualization_data.get_plot_specifications_for_tab_layout().get( + tab_value + ) plot_data = visualization_data.get_current_execution_plot_data( game_id, country_perspective ) diff --git a/stellarisdashboard/dashboard_app/utils.py b/stellarisdashboard/dashboard_app/utils.py index b55c918..3a5fe0f 100644 --- a/stellarisdashboard/dashboard_app/utils.py +++ b/stellarisdashboard/dashboard_app/utils.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) -VERSION = "v2.1" +VERSION = "v3.0" def parse_version(version: str): diff --git a/stellarisdashboard/dashboard_app/visualization_data.py b/stellarisdashboard/dashboard_app/visualization_data.py index 4ab38c9..38c61ec 100644 --- a/stellarisdashboard/dashboard_app/visualization_data.py +++ b/stellarisdashboard/dashboard_app/visualization_data.py @@ -11,7 +11,7 @@ import numpy as np from scipy.spatial import Voronoi -from stellarisdashboard import datamodel, config +from stellarisdashboard import datamodel, config, game_info logger = logging.getLogger(__name__) @@ -39,12 +39,15 @@ class PlotSpecification: # This function specifies which data container class should be used for the plot. # The int argument is the country ID for which budgets and pop stats are shown. data_container_factory: Callable[[Optional[int]], "AbstractPlotDataContainer"] + style: PlotStyle yrange: Tuple[float, float] = None x_axis_label: str = "Time (years after 2200.01.01)" y_axis_label: str = "" + data_container_factory_kwargs: Dict = dataclasses.field(default_factory=dict) + def get_plot_specifications_for_tab_layout(): return { @@ -77,6 +80,7 @@ def get_current_execution_plot_data( for pslist in get_plot_specifications_for_tab_layout().values() for ps in pslist ] + plot_specifications += get_market_graphs(config.CONFIG.market_resources) _CURRENT_EXECUTION_PLOT_DATA[game_name] = PlotDataManager( game_name, plot_specifications ) @@ -103,6 +107,10 @@ def get_color_vals( r, g, b = COLOR_ENGINEERING elif key_str == GalaxyMapData.UNCLAIMED: # for unclaimed system in the galaxy map r, g, b = 255, 255, 255 + elif key_str.endswith("galactic_market"): + r, g, b = 255, 0, 0 + elif key_str.endswith("internal_market"): + r, g, b = 0, 0, 255 else: random.seed(key_str) h = random.uniform(0, 1) @@ -151,7 +159,9 @@ def initialize(self): for plot_spec in self.plot_specifications: self.data_containers_by_plot_id[ plot_spec.plot_id - ] = plot_spec.data_container_factory(self.country_perspective) + ] = plot_spec.data_container_factory( + self.country_perspective, **plot_spec.data_container_factory_kwargs + ) @property def country_perspective(self) -> Optional[int]: @@ -229,7 +239,7 @@ def get_data_for_plot( class AbstractPlotDataContainer(abc.ABC): DEFAULT_VAL = float("nan") - def __init__(self, country_perspective: Optional[int]): + def __init__(self, country_perspective: Optional[int], **kwargs): self.dates: List[float] = [] self.data_dict: Dict[str, List[float]] = {} self._country_perspective = country_perspective @@ -454,6 +464,95 @@ def _iterate_budgetitems( yield "colossi", cd.ship_count_colossus * 32 +class MarketPriceDataContainer(AbstractPlayerInfoDataContainer): + DEFAULT_VAL = float("nan") + galactic_market_indicator_key = "traded_on_galactic_market" + internal_market_indicator_key = "traded_on_internal_market" + + def __init__(self, country_perspective, resource_name, base_price, resource_index): + super().__init__(country_perspective=country_perspective) + self.resource_name = resource_name + self.base_price = base_price + self.resource_index = resource_index + + def _iterate_budgetitems( + self, cd: datamodel.CountryData + ) -> Iterable[Tuple[str, float]]: + gs = cd.game_state + if cd.has_galactic_market_access: + yield from self._iter_galactic_market_price(gs, cd) + else: + yield from self._iter_internal_market_price(gs, cd) + + def _iter_galactic_market_price( + self, gs: datamodel.GameState, cd: datamodel.CountryData + ) -> Iterable[Tuple[str, float]]: + market_fee = self.get_market_fee(gs) + market_resources: List[datamodel.GalacticMarketResource] = sorted( + gs.galactic_market_resources, key=lambda r: r.resource_index + ) + for res, res_data in zip(market_resources, config.CONFIG.market_resources): + if res_data["name"] == self.resource_name and res.availability != 0: + yield from self._get_resource_prices( + market_fee, res_data["base_price"], res.fluctuation + ) + yield self.galactic_market_indicator_key, -0.001 + yield self.internal_market_indicator_key, self.DEFAULT_VAL + break + + def _iter_internal_market_price( + self, gs: datamodel.GameState, cd: datamodel.CountryData + ): + market_fee = self.get_market_fee(gs) + res_data = None + for r in config.CONFIG.market_resources: + if r["name"] == self.resource_name: + res_data = r + break + if res_data is None: + logger.info("Could not find configuration for resource {self.resour") + return + if res_data["base_price"] is None: + return + + always_tradeable = ["minerals", "food", "consumer_goods", "alloys"] + fluctuation = 0.0 if self.resource_name in always_tradeable else None + for resource in cd.internal_market_resources: + if resource.resource_name.text == self.resource_name: + fluctuation = resource.fluctuation + break + if fluctuation is None: + return + + yield from self._get_resource_prices( + market_fee, res_data["base_price"], fluctuation + ) + yield self.galactic_market_indicator_key, self.DEFAULT_VAL + yield self.internal_market_indicator_key, -0.001 + + def _get_resource_prices( + self, market_fee: float, base_price: float, fluctuation: float + ) -> Tuple[float, float, float]: + no_fee_price = base_price * (1 + fluctuation / 100) + buy_price = base_price * (1 + fluctuation / 100) * (1 + market_fee) + sell_price = base_price * (1 + fluctuation / 100) * (1 - market_fee) + + yield f"{self.resource_name}_base_price", no_fee_price + if buy_price != sell_price: + yield f"{self.resource_name}_buy_price", buy_price + yield f"{self.resource_name}_sell_price", sell_price + + def get_market_fee(self, gs): + market_fees = config.CONFIG.market_fee + current_fee = {"date": 0, "fee": 0.3} # default + for fee in sorted(market_fees, key=lambda f: f["date"]): + if datamodel.date_to_days(fee["date"]) > gs.date: + break + current_fee = fee + market_fee = current_fee["fee"] + return market_fee + + class AbstractEconomyBudgetDataContainer(AbstractPlayerInfoDataContainer, abc.ABC): DEFAULT_VAL = 0.0 @@ -464,7 +563,7 @@ def _iterate_budgetitems( val = self._get_value_from_budgetitem(budget_item) if val == 0.0: val = None - yield (budget_item.name, val) + yield budget_item.name, val @abc.abstractmethod def _get_value_from_budgetitem(self, bi: datamodel.BudgetItem) -> float: @@ -1255,6 +1354,37 @@ def _get_value_from_popstats(self, ps: datamodel.PopStatsByStratum): _GALAXY_DATA: Dict[str, "GalaxyMapData"] = {} +def market_graph_id(resource_config) -> str: + return f"trade-price-{resource_config['name']}" + + +def market_graph_title(resource_config) -> str: + resource_name = game_info.convert_id_to_name( + resource_config["name"], remove_prefix="sr" + ) + return f"{resource_name} market price" + + +def get_market_graphs(resource_config) -> List[PlotSpecification]: + res = [] + for idx, rc in enumerate(resource_config): + if rc["base_price"] is not None: + res.append( + PlotSpecification( + plot_id=market_graph_id(rc), + title=market_graph_title(rc), + data_container_factory=MarketPriceDataContainer, + data_container_factory_kwargs=dict( + resource_name=rc["name"], + base_price=rc["base_price"], + resource_index=idx, + ), + style=PlotStyle.line, + ) + ) + return res + + def get_galaxy_data(game_name: str) -> "GalaxyMapData": """Similar to get_current_execution_plot_data, the GalaxyMapData for each game is cached in the _GALAXY_DATA dictionary. diff --git a/stellarisdashboard/datamodel.py b/stellarisdashboard/datamodel.py index 27fd6c4..a749970 100644 --- a/stellarisdashboard/datamodel.py +++ b/stellarisdashboard/datamodel.py @@ -589,6 +589,11 @@ class GameState(Base): country_data = relationship( "CountryData", back_populates="game_state", cascade="all,delete,delete-orphan" ) + galactic_market_resources = relationship( + "GalacticMarketResource", + back_populates="game_state", + cascade="all,delete,delete-orphan", + ) def __str__(self): return f"Gamestate of {self.game.game_name} @ {days_to_date(self.date)}" @@ -982,10 +987,19 @@ class CountryData(Base): has_commercial_pact_with_player = Column(Boolean) is_player_neighbor = Column(Boolean) + has_galactic_market_access = Column(Boolean, default=False) + country = relationship("Country", back_populates="country_data") game_state = relationship("GameState", back_populates="country_data") budget = relationship("BudgetItem", cascade="all,delete,delete-orphan") + + internal_market_resources = relationship( + "InternalMarketResource", + back_populates="country_data", + cascade="all,delete,delete-orphan", + ) + pop_stats_species = relationship( "PopStatsBySpecies", back_populates="country_data", @@ -1050,6 +1064,49 @@ def show_military_info(self): ) +class GalacticMarketResource(Base): + """ + Market data for a single resource at a specific time. + The name of the resource and its + """ + + __tablename__ = "galacticmarketresourcetable" + galactic_market_resource_id = Column(Integer, primary_key=True) + game_state_id = Column(ForeignKey(GameState.gamestate_id), index=True) + country_data_id = Column(ForeignKey(CountryData.country_data_id), index=True) + + # position encodes the resource, matching the order in common/strategic_resources/00_strategic_resources.txt + resource_index = Column(Integer) + availability = Column(Integer) # 0 or 1 + + fluctuation = Column(Float) + + # Buy volume: Sum over all countries + resources_bought = Column(Float) + resources_sold = Column(Float) + + game_state = relationship( + "GameState", + back_populates="galactic_market_resources", + ) + + +class InternalMarketResource(Base): + __tablename__ = "internalmarketresourcetable" + internal_market_resource_id = Column(Integer, primary_key=True) + country_data_id = Column(ForeignKey(CountryData.country_data_id), index=True) + + # internal market resources are stored by name + resource_name_id = Column(ForeignKey(SharedDescription.description_id), index=True) + + fluctuation = Column(Float) + + country_data = relationship( + "CountryData", back_populates="internal_market_resources" + ) + resource_name = relationship("SharedDescription") + + class BudgetItem(Base): __tablename__ = "budgetitemtable" budget_item_id = Column(Integer, primary_key=True) diff --git a/stellarisdashboard/parsing/timeline.py b/stellarisdashboard/parsing/timeline.py index 0139f19..7f19828 100644 --- a/stellarisdashboard/parsing/timeline.py +++ b/stellarisdashboard/parsing/timeline.py @@ -180,6 +180,8 @@ def _data_processors(self) -> Iterable["AbstractGamestateDataProcessor"]: yield DiplomaticRelationsProcessor() yield SensorLinkProcessor() yield CountryDataProcessor() + yield GalacticMarketProcessor() + yield InternalMarketProcessor() yield SpeciesProcessor() yield LeaderProcessor() yield PlanetProcessor() @@ -746,6 +748,10 @@ def extract_data_from_gamestate(self, dependencies): country_id ) + has_market_access = "has_market_access" in country_data_dict.get( + "flags", [] + ) + diplomacy_data = self._get_diplomacy_towards_player( diplomacy_dict, country_id ) @@ -771,6 +777,7 @@ def extract_data_from_gamestate(self, dependencies): economy_power=country_data_dict.get("economy_power", 0), has_sensor_link_with_player=has_sensor_link_with_player, attitude_towards_player=attitude_towards_player, + has_galactic_market_access=has_market_access, # Resource income is calculated below in _extract_country_economy net_energy=0.0, net_minerals=0.0, @@ -2261,6 +2268,101 @@ def _query_event( ) +class GalacticMarketProcessor(AbstractGamestateDataProcessor): + ID = "galactic_market" + DEPENDENCIES = [] + + def extract_data_from_gamestate(self, dependencies): + market = self._gamestate_dict.get("market", {}) + resource_list = market.get("galactic_market_resources", []) + fluctuations = market.get("fluctuations", []) + bought_by_country = market.get("resources_bought", {}).get("amount") + sold_by_country = market.get("resources_sold", {}).get("amount") + + if not all( + [ + market, + resource_list, + fluctuations, + bought_by_country, + sold_by_country, + ] + ): + logger.info( + f"{self._basic_info.logger_str} Missing or invalid Galactic Market data, skipping..." + ) + return + + total_bought = [ + sum(bought[i] for bought in bought_by_country) + for i in range(len(resource_list)) + ] + total_sold = [ + sum(sold[i] for sold in sold_by_country) for i in range(len(resource_list)) + ] + for i, (availability, fluctuation, bought, sold) in enumerate( + zip(resource_list, fluctuations, total_bought, total_sold) + ): + self._session.add( + datamodel.GalacticMarketResource( + game_state=self._db_gamestate, + resource_index=i, + availability=availability, + fluctuation=fluctuation, + resources_bought=bought, + resources_sold=sold, + ) + ) + + +class InternalMarketProcessor(AbstractGamestateDataProcessor): + ID = "internal_market" + DEPENDENCIES = [CountryDataProcessor.ID] + + def extract_data_from_gamestate(self, dependencies): + country_data_dict = dependencies[CountryDataProcessor.ID] + + market = self._gamestate_dict.get("market", {}) + fluctuation_dict = market.get("internal_market_fluctuations", {}) + market_countries = fluctuation_dict.get("country", []) + fluctuation_resources = fluctuation_dict.get("resources", []) + + if not all( + [ + fluctuation_dict, + market_countries, + fluctuation_resources, + self._basic_info.player_country_id in market_countries, + ] + ): + logger.info( + f"{self._basic_info.logger_str} Missing or invalid Internal Market data, skipping..." + ) + return + + for country_id, product_fluctuation in zip( + market_countries, fluctuation_resources + ): + if ( + not isinstance(product_fluctuation, dict) + or country_id not in country_data_dict + ): + continue + if ( + not config.CONFIG.read_all_countries + and country_id != self._basic_info.player_country_id + ): + continue + for name, value in product_fluctuation.items(): + self._session.add( + datamodel.InternalMarketResource( + resource_name=self._get_or_add_shared_description(name), + country_data=country_data_dict[country_id], + fluctuation=value, + ) + ) + + class GalacticCommunityProcessor(AbstractGamestateDataProcessor): ID = "galactic_community" DEPENDENCIES = [CountryProcessor.ID, RulerEventProcessor.ID]