Rust save parser (#97)

.github/workflows/build.yml

@@ -29,9 +29,25 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -r requirements.txt
           python -m pip install -r dev-requirements.txt
-      - name: Build
+      - name: Install rust toolchain
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - name: Run cargo test
+        run: |
+          cargo test
+        working-directory: "stellarisdashboard/parsing/rust_parser"
+      - name: Build rust-parser
+        uses: PyO3/maturin-action@v1
+        with:
+          command: build
+          args: --release -o target
+          working-directory: "stellarisdashboard/parsing/rust_parser"
+      - name: Install rust parser
         run: |
-          python setup.py build_ext --inplace
+          pip install --find-links=stellarisdashboard/parsing/rust_parser/target rust-parser
       - name: Tests
         run: |
           python -m pytest -m "not skip_github_actions"
@@ -41,24 +57,22 @@ jobs:
       - name: pyInstaller run
         run: |
           dist/stellarisdashboard-build/parse_saves
-      # Zip the file, this will lead to a double-zipped artifact but otherwise there are problems with the download
-      # (probably too many individual files?)
+      # Zip the file, makes a zip containing a zip but fixes download problems due to too many small files
       - name: package files (win)
-        if: matrix.os == 'windows-latest' && github.event_name != 'pull_request'
+        if: matrix.os == 'windows-latest'
         run: |
           cd dist
           powershell Compress-Archive stellarisdashboard-build/* ../stellarisdashboard-${{ matrix.os }}.zip
           cd ..
           ls
       - name: package files
-        if: matrix.os != 'windows-latest' && github.event_name != 'pull_request'
+        if: matrix.os != 'windows-latest'
         run: |
           base_dir=`pwd`
           pushd dist/stellarisdashboard-build/
           zip -r $base_dir/stellarisdashboard-${{ matrix.os }}.zip *
           popd
       - name: Publish
-        if: github.event_name != 'pull_request'
         uses: actions/upload-artifact@v3
         with:
           name: stellarisdashboard-${{ matrix.os }}.zip

README.md

@@ -31,16 +31,13 @@ You can also access the dashboard by opening [http://localhost:28053](http://loc
 1. Download the latest release of the dashboard from the [releases page](https://github.com/eliasdoehne/stellaris-dashboard/releases) for your OS.
 2. Extract the zip archive to a location of your choice.
 3. Run the `parse_saves` application to read any existing save files. This will attempt to load any existing save files into the dashboard's database.
-4. Run the `stellarisdashboard` application. 
-
-On Linux (and mac?) you should run the applications from a terminal otherwise there won't be any output and it can be difficult to tell if it is working. Looking into improving this in the future.
-
-I unfortunately don't have access to a mac and can't check every release on all platforms as thoroughly as I'd like... If there are problems, please make discussion thread on Steam or open a new issue on github.
+4. Run the `stellarisdashboard` application. (recommended to run from a terminal in Linux and macOs to see the dashboard output)
 
 ### Build it yourself
 - Get Python 3.10 (it may work on other versions too)
 - (Recommended) create & activate a virtual environment 
 - `pip install .`
+- `pip install maturin` and `maturin develop -r` in `stellarisdashboard/parsing/rust_parser`
 - `stellarisdashboard`
 
 # Other information

dev-requirements.txt

@@ -1,4 +1,4 @@
 pytest
-Cython
 pyinstaller
 black==22.3.0
+maturin

pyinstaller-package.spec

@@ -17,7 +17,6 @@ added_files = [
     (str(Path("README.md")), "./"),
     (str(Path("stellarisdashboard/dashboard_app/assets")), "stellarisdashboard/dashboard_app/assets/"),
     (str(Path("stellarisdashboard/dashboard_app/templates")), "stellarisdashboard/dashboard_app/templates/"),
-    (str(Path("stellarisdashboard/parsing/cython_ext")), "stellarisdashboard/parsing/cython_ext/"),
 ]
 
 a_main = Analysis(

requirements.txt

@@ -29,4 +29,4 @@ SQLAlchemy==1.4.46
 tenacity==8.1.0
 tqdm==4.64.1
 waitress==2.1.2
-Werkzeug==2.2.2
+Werkzeug==2.2.3

setup.py

@@ -24,27 +24,12 @@
 
 from setuptools import setup
 
-try:
-    # Try to build the cython extension locally
-    from Cython.Build import cythonize
-
-    extension_modules = cythonize("stellarisdashboard/parsing/cython_ext/tokenizer.pyx")
-except ImportError:
-    print(
-        "Cython is not installed, using pre-built C-extension if available, or (slow) fallback solution."
-    )
-    extension_modules = []
-except RuntimeError as e:
-    print(f"Warning: RuntimeError while building Cython extension: {e}")
-    print("Using pre-built C-extension if available, or (slow) fallback solution.")
-    extension_modules = []
-
 with open("requirements.txt", "r") as f:
     install_requires = f.read().strip().split()
 
 setup(
     name="stellarisdashboard",
-    ext_modules=extension_modules,
+    ext_modules=[],
     install_requires=install_requires,
     entry_points={
         "console_scripts": [

stellarisdashboard/config.py

@@ -1,7 +1,6 @@
 import dataclasses
 import itertools
 import logging
-import multiprocessing as mp  # only to get the cpu count
 import pathlib
 import platform
 import sys
@@ -12,10 +11,9 @@
 import yaml
 
 LOG_LEVELS = {"CRITICAL": logging.CRITICAL, "ERROR": logging.ERROR, "WARNING": logging.WARNING, "INFO": logging.INFO, "DEBUG": logging.DEBUG}
-CPU_COUNT = mp.cpu_count()
 
 LOG_FORMAT = logging.Formatter(
-    "%(processName)s - %(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
 )
 
 CONFIG: "Config" = None
@@ -32,16 +30,6 @@ def initialize_logger():
     root_logger.addHandler(stdout_ch)
 
 
-def _get_default_thread_count():
-    if CPU_COUNT < 4:
-        threads = 1
-    elif CPU_COUNT == 4:
-        threads = 2
-    else:
-        threads = max(1, CPU_COUNT // 2 - 1)
-    return threads
-
-
 def _get_default_save_path():
     # according to https://stellaris.paradoxwikis.com/Save-game_editing
     home = pathlib.Path.home()
@@ -177,7 +165,7 @@ def _get_default_base_output_path():
     save_file_path=_get_default_save_path(),
     mp_username="",
     base_output_path=_get_default_base_output_path(),
-    threads=_get_default_thread_count(),
+    threads=1,
     localization_file_dir=_get_default_localization_file_dir(),
     host="127.0.0.1",
     port=28053,

stellarisdashboard/dashboard_app/settings.py

@@ -48,7 +48,6 @@ def _bool_to_lowercase(py_bool: bool) -> str:
             "type": t_int,
             "value": current_values["threads"],
             "min": 1,
-            "max": config.CPU_COUNT,
             "name": "Number of threads (applies after restart)",
             "description": "Number of threads for reading save files, updates apply after restarting the dashboard.",
         },

stellarisdashboard/parsing/rust_parser/.gitignore

@@ -0,0 +1,72 @@
+/target
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+.pytest_cache/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.venv/
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+include/
+man/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+.DS_Store
+
+# Sphinx documentation
+docs/_build/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Pyenv
+.python-version