refactor elfpy to look like a monorepo

.github/workflows/lint.yml

@@ -28,7 +28,8 @@ jobs:
           token: ${{ secrets.GH_GITHUB_COM_TOKEN || github.token }}
       - run: |
           python -m pip install --upgrade pip
-          python -m pip install .[with-dependencies,ape]
+          python -m pip install -r requirements.txt
+          python -m pip install -r requirements-dev.txt
 
       - name: Ensure clear Jupyter Notebooks
         uses: ResearchSoftwareActions/[email protected]

.github/workflows/static.yml

@@ -28,7 +28,8 @@ jobs:
           token: ${{ secrets.GH_GITHUB_COM_TOKEN || github.token}}
       - run: |
           python -m pip install --upgrade pip
-          python -m pip install .[with-dependencies,ape]
+          python -m pip install -r requirements.txt
+          python -m pip install -r requirements-dev.txt
 
       - name: Analysing code with pyright
         run: python -m pyright $(git ls-files '*.py' '*.pyi')

.github/workflows/test.yml

@@ -47,7 +47,8 @@ jobs:
           token: ${{ secrets.GH_GITHUB_COM_TOKEN || github.token}}
       - run: |
           python -m pip install --upgrade pip
-          python -m pip install .[with-dependencies]
+          python -m pip install -r requirements.txt
+          python -m pip install -r requirements-dev.txt
           python -m pip install coverage
 
       - name: Run pytest with coverage

docs/source/conf.py

@@ -16,7 +16,11 @@
 import tomli
 
 # indicate where the elfpy Python package lives
-elfpy_root = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))
+elfpy_root = os.path.join(
+    os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))),
+    "lib",
+    "elfpy",
+)
 sys.path.insert(0, os.path.abspath("."))
 sys.path.insert(0, elfpy_root)
 
@@ -138,9 +142,9 @@ def _get_project_meta():
 
 # -- Options for API document generation -------------------------------------------------
 
-autoapi_dirs = ["../../elfpy"]
+autoapi_dirs = [os.path.join("..", "..", "lib", "elfpy")]
 autoapi_type = "python"
-autoapi_template_dir = "_templates/autoapi"
+autoapi_template_dir = os.path.join("_templates", "autoapi")
 autoapi_options = [
     "members",
     "undoc-members",

lib/elfpy/BUILD.md

@@ -0,0 +1,42 @@
+# Build
+
+We use Docker to manage our build environment.
+
+This requires access to the private hyperdrive repo, available only to Delv team members currently.
+
+To build a local image, make sure you have Docker installed and running, then use:
+
+```bash
+chmod +x build_local_docker_image.sh
+./build_local_docker_image.sh
+```
+
+If built successfully, a `docker image ls` command should look like this:
+
+```
+REPOSITORY    TAG       IMAGE ID       CREATED          SIZE
+elf-sims      latest    fd84351979d7   21 minutes ago   2.04GB
+```
+
+This image receives the `elf-sims` tag and can be reference with`image: elf-sims:latest`,
+for instance if you're using Docker Compose.
+
+To build a local image and push it to ghcr.io, make sure you have Docker installed and running, then use:
+
+```bash
+>> ./scripts/build_and_push_docker_iamge.sh
+Tag pushed to ghcr.io/delvtech/elf-simulations/dcdefc6153fc
+Run the following command to retrieve the image:
+docker image pull ghcr.io/delvtech/elf-simulations/dcdefc6153fc
+```
+
+If you get a 401 error, you'll need to make a GH Personal Access Token with write access.
+
+You can optionally provide a tag
+
+```bash
+>> ./scripts/build_and_push_docker_iamge.sh matt-latest
+Tag pushed to ghcr.io/delvtech/elf-simulations/matt-latest
+Run the following command to retrieve the image:
+docker image pull ghcr.io/delvtech/elf-simulations/matt-latest
+```

lib/elfpy/Dockerfile

@@ -0,0 +1,21 @@
+### Migrations Image ###
+# pinned to a specific image chosen from https://github.com/delvtech/hyperdrive/pkgs/container/hyperdrive%2Fdevnet
+FROM ghcr.io/delvtech/hyperdrive/devnet:0.0.8 as migrations
+
+# ### Python Image ###
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# copy everything in elf-simulations
+COPY . ./
+
+# install elfpy in one step, adding build tools, then removing them
+# https://stackoverflow.com/questions/58300046/how-to-make-lightweight-docker-image-for-python-app-with-pipenv
+RUN python -m pip install --no-cache-dir --upgrade pip && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends gcc python3-dev libssl-dev git && \
+  python -m pip install --no-cache-dir -e ."[with-dependencies,ape]" && \
+  apt-get remove -y gcc python3-dev libssl-dev && \
+  apt-get autoremove -y && \
+  pip uninstall pipenv -y

lib/elfpy/README.md

@@ -0,0 +1,94 @@
+[![](https://codecov.io/gh/delvtech/elf-simulations/branch/main/graph/badge.svg?token=1S60MD42ZP)](https://app.codecov.io/gh/delvtech/elf-simulations?displayType=list)
+[![](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![](https://img.shields.io/badge/testing-pytest-blue.svg)](https://docs.pytest.org/en/latest/contents.html)
+<br><a href="https://app.codecov.io/gh/delvtech/elf-simulations?displayType=list"><img height="50px" src="https://codecov.io/gh/delvtech/elf-simulations/branch/main/graphs/sunburst.svg?token=1S60MD42ZP"><a>
+
+# [DELV](https://delv.tech) market simulation and analysis
+
+This project is a work-in-progress. All code is provided as is and without guarantee.
+The language used in this code and documentation is not intended to, and does not, have any particular financial, legal, or regulatory significance.
+
+Documentation can be found [here](https://elfpy.delv.tech).
+
+## Install
+
+Please refer to [INSTALL.md](https://github.com/delvtech/elf-simulations/blob/main/INSTALL.md).
+
+## Deployment
+
+Please refer to [BUILD.md](https://github.com/delvtech/elf-simulations/blob/main/BUILD.md).
+
+## Testing
+
+Testing is achieved with [py.test](https://docs.pytest.org/en/latest/contents.html). You can run all tests from the repository root directory by running `python -m pytest`, or you can pick a specific test in the `tests/` folder with `python -m pytest tests/{test_file.py}`.
+
+## Coverage
+
+To run coverage locally you can follow these steps:
+
+```bash
+pip install coverage
+coverage run -m pytest
+coverage html
+```
+
+then just open `htmlcov/index.html` to view the report!
+
+## Examples
+
+Python files in the `examples/` folder should be executable from the repository root. Run them with the -h flag to see argument options. The Jupyter notebooks contained in `examples/notebooks/` should be run locally using [Jupyter](https://jupyter.org/install), [VS Code](https://code.visualstudio.com/docs/datascience/jupyter-notebooks), or something equivalent.
+
+## Contributions
+
+Please refer to [CONTRIBUTING.md](https://github.com/delvtech/elf-simulations/blob/main/CONTRIBUTING.md).
+
+## Number format
+
+Internally Elfpy conducts all operations using 18-decimal fixed-point precision integers and arithmetic.
+Briefly, this means our representation for unity, "one", is `1 * 10 ** 18`, which would be `1.0` when cast to a float.
+
+This can lead to confusion when additionally dealing with standard Python floats and ints.
+As such, we have purposefully constrain support for mixed-type operations that include the FixedPoint type.
+Due to a lack of known precision, operations against Python floats are not allowed (e.g. `float * FixedPoint` will raise an error).
+However, operations against `int` are allowed.
+In this case, the `int` _argument_ is assumed to be "unscaled", i.e. if you write `int(8) * FixedPoint(8)` we will scale up the first variable return a `FixedPoint` number that represents the float `64.0` in 18-decimal FixedPoint format.
+So in this example the internal representation of that operation would be `64*10**18`.
+If you cast FixedPoint numbers to ints or floats you will get "unscaled" representation, e.g. `float(FixedPoint(8.0)) == 8.0` and `int(FixedPoint(8.528)) == 8`.
+
+If you want the integer scaled representation, which can be useful for communicating with Solidity contracts, you must ask for it explicitly, e.g. `FixedPoint(8.52).scaled_value == 8520000000000000000`.
+Conversely, if you want to initialize a FixedPoint variable using the scaled integer representation, then you need to instantiate the variable using the `scaled_value` argument, e.g. `FixedPoint(scaled_value=8)`.
+In that example, the internal representation is `8`, so casting it to a float would produce a small value: `float(FixedPoint(scaled_value=8)) == 8e-18`.
+
+To understand more, we recommend that you study the fixed point tests and source implementation in `elfpy/math/`.
+
+## Modifying configuration for bot deployment
+Bots can be deployed using the `evm_bots.py` script in the examples folder.
+If you wish to run it with non-default parameters, then you must specify them as a JSON file.
+To generate a default configuration file, from the base directory, run:
+
+```bash
+python elfpy/bots/bots_default_config.py
+```
+
+This will generate `bots_config.default.json`, which you can feed into `evm_bots.py`:
+
+```bash
+evm_bots.py bots_config.default.json
+```
+
+## Data pipeline
+The data pipeline queries the running chain and exports data to a postgres database. The `infra` repository spins up a local postgres instance via Docker, and the data pipeline will point to this by default. Optionally, you can also configure the backend database by specifying the following environmental variables (for example, in a `.env` file in the base of the repo):
+
+```bash
+POSTGRES_USER="admin"
+POSTGRES_PASSWORD="password"
+POSTGRES_DB="postgres_db"
+POSTGRES_HOST="localhost"
+POSTGRES_PORT=5432
+```
+
+The data script can be then ran using the following command:
+
+```bash
+python elfpy/data/acquire_data.py
+```

tests/pricing_models/test_calc_in_given_out.py → lib/elfpy/elfpy/tests/pricing_models/test_calc_in_given_out.py

@@ -6,15 +6,6 @@
 import unittest
 from typing import TYPE_CHECKING
 
-from calc_test_dataclasses import (
-    CalcInGivenOutFailureTestCase,
-    CalcInGivenOutSuccessByModelTestResult,
-    CalcInGivenOutSuccessTestCase,
-    CalcInGivenOutSuccessTestResult,
-)
-from fixedpointmath import FixedPoint
-from fixedpointmath import errors as fperrors
-
 import elfpy.time as time
 import elfpy.types as types
 import elfpy.utils.logs as log_utils
@@ -24,6 +15,14 @@
     YieldspacePricingModel,
     hyperdrive_actions,
 )
+from elfpy.tests.pricing_models.calc_test_dataclasses import (
+    CalcInGivenOutFailureTestCase,
+    CalcInGivenOutSuccessByModelTestResult,
+    CalcInGivenOutSuccessTestCase,
+    CalcInGivenOutSuccessTestResult,
+)
+from fixedpointmath import FixedPoint
+from fixedpointmath import errors as fperrors
 
 if TYPE_CHECKING:
     from elfpy.markets.base import BasePricingModel

tests/pricing_models/test_calc_max_long_short.py → lib/elfpy/elfpy/tests/pricing_models/test_calc_max_long_short.py

@@ -4,12 +4,11 @@
 import unittest
 from dataclasses import dataclass
 
-from fixedpointmath import FixedPoint
-
 import elfpy.time as time
 import elfpy.utils.logs as log_utils
 from elfpy.markets.hyperdrive import HyperdriveMarketState, HyperdrivePricingModel
-from tests.fixtures.hyperdrive_config import HyperdriveConfig
+from elfpy.tests.fixtures.hyperdrive_config import HyperdriveConfig
+from fixedpointmath import FixedPoint
 
 
 @dataclass

tests/pricing_models/test_calc_out_given_in.py → lib/elfpy/elfpy/tests/pricing_models/test_calc_out_given_in.py

@@ -4,15 +4,6 @@
 import logging
 import unittest
 
-from calc_test_dataclasses import (
-    CalcOutGivenInFailureTestCase,
-    CalcOutGivenInSuccessByModelTestResult,
-    CalcOutGivenInSuccessTestCase,
-    CalcOutGivenInSuccessTestResult,
-)
-from fixedpointmath import FixedPoint
-from fixedpointmath import errors as fperrors
-
 import elfpy.time as time
 import elfpy.types as types
 import elfpy.utils.logs as log_utils
@@ -23,6 +14,14 @@
     YieldspacePricingModel,
     hyperdrive_actions,
 )
+from elfpy.tests.pricing_models.calc_test_dataclasses import (
+    CalcOutGivenInFailureTestCase,
+    CalcOutGivenInSuccessByModelTestResult,
+    CalcOutGivenInSuccessTestCase,
+    CalcOutGivenInSuccessTestResult,
+)
+from fixedpointmath import FixedPoint
+from fixedpointmath import errors as fperrors
 
 # pylint: disable=too-many-lines
 

lib/elfpy/pyproject.toml

@@ -0,0 +1,46 @@
+[project]
+name = "elfpy"
+version = "0.3.2"
+authors = [
+    { name = "Dylan Paiton", email = "[email protected]" },
+    { name = "Mihai Cosma", email = "[email protected]" },
+    { name = "Jonny Rhea", email = "[email protected]" },
+    { name = "Matthew Brown", email = "[email protected]" },
+    { name = "Alex Towle", email = "[email protected]" },
+    { name = "Sheng Lundquist", email = "[email protected]" },
+    { name = "Patrick Morris", email = "[email protected]" },
+    { name = "Giovanni Effio", email = "[email protected]" },
+    { name = "Ryan Goree", email = "[email protected]" },
+    { name = "Will Villanueva", email = "[email protected]" },
+    { name = "Jacob Arruda", email = "[email protected]" },
+    { name = "Violet Vienhage", email = "[email protected]" },
+]
+description = "Trading bots, market simulators, and experiment management by Delv"
+readme = "README.md"
+requires-python = ">=3.10, <3.11"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: Apache License 2.0",
+    "Operating System :: OS Independent",
+]
+
+dependencies = [
+    # Install fixedpointmath from github
+    # Note that this is pointing to main, so if any changes are made to fixedpointmath, we'll have to
+    # upgrade this package
+    "fixedpointmath @ git+https://github.com/delvtech/agent_0.git/#subdirectory=lib/fixedpointmath",
+    "matplotlib",
+    "numpy",
+    "pandas",
+    "python-dotenv",
+    "scipy",
+    "typing_extensions==4.5.0",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/delvtech/elf-simulations"
+"Bug Tracker" = "https://github.com/delvtech/elf-simulations/issues"
+
+[build-system]
+requires = ["flit_core>=3.2"]
+build-backend = "flit_core.buildapi"

examples/notebooks/hyperdrive.ipynb → notebooks/hyperdrive.ipynb

@@ -534,7 +534,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.11.1"
+      "version": "3.10.12"
     },
     "vscode": {
       "interpreter": {