Add bindings for StepData types, add operator[] api and bindings with…

… __getitem__ for replay data, add basic python api tests, add python documentation, rename doc folder

.github/workflows/readthedocs.yml

@@ -2,24 +2,24 @@
 name: Docs
 on:
   push:
-    branches: [ main ]
+    branches: [main]
 jobs:
   build:
     runs-on: ubuntu-24.04
     steps:
-    - name: Requirements
-      run: |
-        sudo apt-get update
-        sudo apt-get install doxygen sphinx-doc python3-sphinx-rtd-theme python3-breathe python3-sphinx-sitemap
-    - name: Checkout repo
-      uses: actions/checkout@v4
-    - name: Build docs
-      run: |
-        make -C docs_sphinx html
-        touch docs_sphinx/_build/html/.nojekyll
-    - name: Deploy
-      uses: JamesIves/github-pages-deploy-action@releases/v3
-      with:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        BRANCH: gh-pages # The branch the action should deploy to.
-        FOLDER: docs_sphinx/_build/html # The folder the action should deploy.
+      - name: Requirements
+        run: |
+          sudo apt-get update
+          sudo apt-get install doxygen sphinx-doc python3-sphinx-rtd-theme python3-breathe
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Build docs
+        run: |
+          make -C docs_sphinx html
+          touch docs_sphinx/_build/html/.nojekyll
+      - name: Deploy
+        uses: JamesIves/github-pages-deploy-action@releases/v3
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BRANCH: gh-pages # The branch the action should deploy to.
+          FOLDER: docs_sphinx/_build/html # The folder the action should deploy.

.gitignore

@@ -168,7 +168,6 @@ src/sc2_replay_reader/_sc2_replay_reader.pyi
 # Visual Studio Fragments
 .vs
 out
-docs/
 _deps/
 _build/
 _static/

docs_sphinx/conf.py → docs/conf.py

@@ -11,9 +11,11 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 # import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+import sys
 import subprocess
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "src" / "sc2_replay_reader"))
 
 # Doxygen
 subprocess.call("doxygen Doxyfile", shell=True)
@@ -39,8 +41,8 @@
     "sphinx.ext.mathjax",
     "sphinx.ext.ifconfig",
     "sphinx.ext.viewcode",
-    "sphinx_sitemap",
     "sphinx.ext.inheritance_diagram",
+    "sphinx.ext.napoleon",
     "breathe",
 ]
 

docs/dataloading.rst

@@ -0,0 +1,23 @@
+Dataloading
+===========
+
+Other than Python bindings to the C++ Replay Database and Structures, we include some basic dataloading utilities and an example for pytorch.
+
+ReplaySampler
+-------------
+
+.. autoclass:: sc2_replay_reader.sampler.ReplaySampler
+    :members:
+
+.. autoclass:: sc2_replay_reader.sampler.BasicSampler
+    :members:
+
+.. autoclass:: sc2_replay_reader.sampler.SQLSampler
+    :members:
+
+
+PyTorch Example
+---------------
+
+.. autoclass:: sc2_replay_reader.example.SC2Dataset
+    :members:

docs_sphinx/index.rst → docs/index.rst

@@ -22,6 +22,7 @@ Table of Contents
     :maxdepth: 2
 
     self
-    cpp_api/index
     replay_data
+    dataloading
     benchmark
+    cpp_api/index

include/data_structures/replay_all.hpp

@@ -104,6 +104,13 @@ struct StepDataSoA
         stepData.neutralUnits = neutralUnits[idx];
         return stepData;
     }
+
+    /**
+     * @brief Number of game steps in the Structure-of-Arrays
+     *
+     * @return std::size_t
+     */
+    [[nodiscard]] auto size() const noexcept -> std::size_t { return gameStep.size(); }
 };
 
 static_assert(

include/data_structures/replay_interface.hpp

@@ -158,6 +158,29 @@ template<typename StepDataType> struct ReplayDataTemplate
         return header == other.header && data == other.data;
     }
 
+    /**
+     * @brief Number of observed steps in the replay data
+     *
+     * @return std::size_t
+     */
+    [[nodiscard]] auto size() const noexcept -> std::size_t { return data.size(); }
+
+    /**
+     * @brief Get constant reference to replay data at step index
+     *
+     * @param index index of replay observations
+     * @return const StepDataType&
+     */
+    [[nodiscard]] auto operator[](std::size_t index) const noexcept -> const StepDataType & { return data[index]; }
+
+    /**
+     * @brief Get reference to replay data at step index
+     *
+     * @param index index of replay observations
+     * @return const StepDataType&
+     */
+    [[nodiscard]] auto operator[](std::size_t index) noexcept -> StepDataType & { return data[index]; }
+
     /**
      * @brief Get the unique hash ID of the replay
      * @return Mutable reference of string hash of the replay
@@ -191,9 +214,9 @@ template<typename StepDataType> struct ReplayDataTemplate
 template<IsSoAType StepDataSoAType> struct ReplayDataTemplateSoA
 {
     /**
-     * @brief type of replay data
+     * @brief struct type of observation data at each step
      */
-    using step_type = StepDataSoAType;
+    using step_type = StepDataSoAType::struct_type;
 
     /**
      * @brief AoS equivalent struct type
@@ -220,6 +243,29 @@ template<IsSoAType StepDataSoAType> struct ReplayDataTemplateSoA
         return header == other.header && data == other.data;
     }
 
+    /**
+     * @brief Number of observed steps in the replay data
+     *
+     * @return std::size_t
+     */
+    [[nodiscard]] auto size() const noexcept -> std::size_t { return data.size(); }
+
+    /**
+     * @brief Get constant reference to replay data at step index
+     *
+     * @param index index of replay observations
+     * @return const StepDataType&
+     */
+    [[nodiscard]] auto operator[](std::size_t index) const noexcept -> const step_type & { return data[index]; }
+
+    /**
+     * @brief Get reference to replay data at step index
+     *
+     * @param index index of replay observations
+     * @return const StepDataType&
+     */
+    [[nodiscard]] auto operator[](std::size_t index) noexcept -> step_type & { return data[index]; }
+
     /**
      * @brief Get the unique hash ID of the replay
      * @return Mutable reference of string hash of the replay
@@ -243,13 +289,6 @@ template<IsSoAType StepDataSoAType> struct ReplayDataTemplateSoA
      * @return Constant reference to Player Id of the replay
      */
     [[nodiscard]] auto getPlayerId() const noexcept -> std::uint32_t { return header.playerId; }
-
-    /**
-     * @brief Gather the step data from struct of arrays at a point in time.
-     * @param idx Index of replay to gather data
-     * @return Observation data in original struct form
-     */
-    [[nodiscard]] auto operator[](std::size_t idx) const noexcept -> StepDataSoAType::struct_type { return data[idx]; }
 };
 
 /**

include/data_structures/replay_minimaps.hpp

@@ -48,7 +48,7 @@ static_assert(HasScalarData<StepDataNoUnits> && HasMinimapData<StepDataNoUnits>)
 /**
  * @brief SoA representation of an array of StepDataNoUnits
  */
-struct StepDataSoANoUnits
+struct StepDataNoUnitsSoA
 {
     using has_scalar_data = std::true_type;
     using has_minimap_data = std::true_type;
@@ -68,7 +68,7 @@ struct StepDataSoANoUnits
     std::vector<Image<bool>> buildable{};
     std::vector<Image<bool>> pathable{};
 
-    [[nodiscard]] auto operator==(const StepDataSoANoUnits &other) const noexcept -> bool = default;
+    [[nodiscard]] auto operator==(const StepDataNoUnitsSoA &other) const noexcept -> bool = default;
 
     /**
      * @brief Gather step data from each array to make structure of data at step.
@@ -93,9 +93,16 @@ struct StepDataSoANoUnits
         stepData.pathable = pathable[idx];
         return stepData;
     }
+
+    /**
+     * @brief Number of game steps in the Structure-of-Arrays
+     *
+     * @return std::size_t
+     */
+    [[nodiscard]] auto size() const noexcept -> std::size_t { return gameStep.size(); }
 };
 
-static_assert(HasScalarData<StepDataSoANoUnits> && HasMinimapData<StepDataSoANoUnits> && IsSoAType<StepDataSoANoUnits>);
+static_assert(HasScalarData<StepDataNoUnitsSoA> && HasMinimapData<StepDataNoUnitsSoA> && IsSoAType<StepDataNoUnitsSoA>);
 
 /**
  * @brief ReplayData with minimap and scalar data
@@ -105,7 +112,7 @@ using ReplayDataNoUnits = ReplayDataTemplate<StepDataNoUnits>;
 /**
  * @brief ReplayData as SoA with minimap and scalar data
  */
-using ReplayDataSoANoUnits = ReplayDataTemplateSoA<StepDataSoANoUnits>;
+using ReplayDataSoANoUnits = ReplayDataTemplateSoA<StepDataNoUnitsSoA>;
 
 static_assert(std::same_as<ReplayDataSoANoUnits::struct_type, ReplayDataNoUnits>);
 

include/data_structures/replay_scalars.hpp

@@ -22,7 +22,7 @@ namespace cvt {
 /**
  * @brief Replay step data that only contains scalar data
  */
-struct StepDataNoUnitsMiniMap
+struct StepDataNoUnitsMinimap
 {
     using has_scalar_data = std::true_type;
 
@@ -34,19 +34,19 @@ struct StepDataNoUnitsMiniMap
     std::uint16_t popWorkers{};
     Score score{};
 
-    [[nodiscard]] auto operator==(const StepDataNoUnitsMiniMap &other) const noexcept -> bool = default;
+    [[nodiscard]] auto operator==(const StepDataNoUnitsMinimap &other) const noexcept -> bool = default;
 };
 
-static_assert(HasScalarData<StepDataNoUnitsMiniMap>);
+static_assert(HasScalarData<StepDataNoUnitsMinimap>);
 
 /**
- * @brief SoA representation of an array of StepDataNoUnitsMiniMap
+ * @brief SoA representation of an array of StepDataNoUnitsMinimap
  */
-struct StepDataSoANoUnitsMiniMap
+struct StepDataNoUnitsMinimapSoA
 {
     using has_scalar_data = std::true_type;
 
-    using struct_type = StepDataNoUnitsMiniMap;
+    using struct_type = StepDataNoUnitsMinimap;
 
     std::vector<std::uint32_t> gameStep{};
     std::vector<std::uint16_t> minearals{};
@@ -56,16 +56,16 @@ struct StepDataSoANoUnitsMiniMap
     std::vector<std::uint16_t> popWorkers{};
     std::vector<Score> score{};
 
-    [[nodiscard]] auto operator==(const StepDataSoANoUnitsMiniMap &other) const noexcept -> bool = default;
+    [[nodiscard]] auto operator==(const StepDataNoUnitsMinimapSoA &other) const noexcept -> bool = default;
 
     /**
      * @brief Gather step data from each array to make structure of data at step.
      * @param idx time index of replay to gather.
      * @return Gathered step data.
      */
-    [[nodiscard]] auto operator[](std::size_t idx) const noexcept -> StepDataNoUnitsMiniMap
+    [[nodiscard]] auto operator[](std::size_t idx) const noexcept -> StepDataNoUnitsMinimap
     {
-        StepDataNoUnitsMiniMap stepData;
+        StepDataNoUnitsMinimap stepData;
         stepData.gameStep = gameStep[idx];
         stepData.minearals = minearals[idx];
         stepData.vespene = vespene[idx];
@@ -75,19 +75,26 @@ struct StepDataSoANoUnitsMiniMap
         stepData.score = score[idx];
         return stepData;
     }
+
+    /**
+     * @brief Number of game steps in the Structure-of-Arrays
+     *
+     * @return std::size_t
+     */
+    [[nodiscard]] auto size() const noexcept -> std::size_t { return gameStep.size(); }
 };
 
-static_assert(HasScalarData<StepDataSoANoUnitsMiniMap> && IsSoAType<StepDataSoANoUnitsMiniMap>);
+static_assert(HasScalarData<StepDataNoUnitsMinimapSoA> && IsSoAType<StepDataNoUnitsMinimapSoA>);
 
 /**
  * @brief ReplayData with minimap and scalar data
  */
-using ReplayDataNoUnitsMiniMap = ReplayDataTemplate<StepDataNoUnitsMiniMap>;
+using ReplayDataNoUnitsMiniMap = ReplayDataTemplate<StepDataNoUnitsMinimap>;
 
 /**
  * @brief ReplayData as SoA with minimap and scalar data
  */
-using ReplayDataSoANoUnitsMiniMap = ReplayDataTemplateSoA<StepDataSoANoUnitsMiniMap>;
+using ReplayDataSoANoUnitsMiniMap = ReplayDataTemplateSoA<StepDataNoUnitsMinimapSoA>;
 static_assert(std::same_as<ReplayDataSoANoUnitsMiniMap::struct_type, ReplayDataNoUnitsMiniMap>);
 
 /**