diff --git a/.github/workflows/platform-api-integration-test.yml b/.github/workflows/platform-api-integration-test.yml index def45202ec60..0ca94380575e 100644 --- a/.github/workflows/platform-api-integration-test.yml +++ b/.github/workflows/platform-api-integration-test.yml @@ -102,8 +102,8 @@ jobs: pip list uvicorn openbb_core.api.rest_api:app --host 0.0.0.0 --port 8000 --reload & - - name: Wait for 10 seconds - run: sleep 30 + - name: Wait for 42 seconds + run: sleep 42 - name: Run API Integration Tests run: | diff --git a/.github/workflows/pypi-nightly.yml b/.github/workflows/pypi-nightly.yml index 4cd72a4ccdf4..a6ad0d3bbc69 100644 --- a/.github/workflows/pypi-nightly.yml +++ b/.github/workflows/pypi-nightly.yml @@ -12,37 +12,52 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Setup Python 3.9 - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Update version in pyproject.toml and Edit docs - run: | - sed -i 's/name = ".*"/name = "openbb-nightly"/' pyproject.toml - sed -i "3s/version = \"\(.*\)\"/version = \"\1.dev$(date +%Y%m%d)\"/" pyproject.toml - head pyproject.toml - sed -i 's/pip install openbb/pip install openbb-nightly/g; s/from openbb_terminal.sdk import openbb/from openbb_terminal.sdk import openbb/g' ./website/pypi.md - - - - name: Install pypa/build - run: >- - python -m - pip install - build - --user - - - name: Build a binary wheel and a source tarball - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - . - - - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.NIGHTLY_PYPI_API_TOKEN }} + - uses: actions/checkout@v3 + - name: Setup Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Install pypa/build + run: >- + python -m + pip install + build + --user + + - name: OpenBBTerminal - Update version in pyproject.toml and Edit docs + run: | + sed -i 's/name = ".*"/name = "openbb-terminal-nightly"/' pyproject.toml + sed -i "3s/version = \"\(.*\)\"/version = \"\1.dev$(date +%Y%m%d%H%M)\"/" pyproject.toml + sed -i 's|pip install openbb-terminal|pip install openbb-terminal-nightly|g' ./website/pypi.md + + - name: OpenBBTerminal - Build a binary wheel and a source tarball + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ + . + + - name: OpenBBTerminal - Publish distribution 📦 to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + password: ${{ secrets.TEST_PYPI_API_TOKEN_OBB_TERMINAL_NIGHTLY }} + + - name: OpenBB Platform - Update version in pyproject.toml and Edit docs + run: | + sed -i 's/name = ".*"/name = "openbb-nightly"/' openbb_platform/pyproject.toml + sed -i "3s/version = \"\(.*\)\"/version = \"\1.dev$(date +%Y%m%d%H%M)\"/" openbb_platform/pyproject.toml + + - name: OpenBB Platform - Create the dinamically generated wheel + run: | + python -m pip install poetry toml + python build/pypi/openbb_platform/nightly.py + + - name: OpenBB Platform - Publish distribution 📦 to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: openbb_platform/dist/ + password: ${{ secrets.NIGHTLY_PYPI_API_TOKEN }} diff --git a/.github/workflows/test_pypi_platform.yml b/.github/workflows/test_pypi_platform.yml new file mode 100644 index 000000000000..54fe09f7dc24 --- /dev/null +++ b/.github/workflows/test_pypi_platform.yml @@ -0,0 +1,68 @@ +name: Deploy the OpenBB Platform and the OpenBBTerminal to Test PyPI + +on: + push: + branches: + - feature/v4-pypi + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + deploy-test-pypi: + name: Build and publish 📦 to TestPyPI + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Setup Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Install pypa/build + run: >- + python -m + pip install + build + --user + + - name: OpenBBTerminal - Update version in pyproject.toml and Edit docs + run: | + sed -i 's/name = ".*"/name = "openbb-terminal-nightly"/' pyproject.toml + sed -i "3s/version = \"\(.*\)\"/version = \"\1.dev$(date +%Y%m%d%H%M)\"/" pyproject.toml + sed -i 's|pip install openbb-terminal|pip install openbb-terminal-nightly|g' ./website/pypi.md + + - name: OpenBBTerminal - Build a binary wheel and a source tarball + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ + . + + - name: OpenBBTerminal - Publish distribution 📦 to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + password: ${{ secrets.TEST_PYPI_API_TOKEN_OBB_TERMINAL_NIGHTLY }} + repository-url: https://test.pypi.org/legacy/ + + - name: OpenBB Platform - Update version in pyproject.toml and Edit docs + run: | + sed -i 's/name = ".*"/name = "openbb-nightly"/' openbb_platform/pyproject.toml + sed -i "3s/version = \"\(.*\)\"/version = \"\1.dev$(date +%Y%m%d%H%M)\"/" openbb_platform/pyproject.toml + + - name: OpenBB Platform - Create the dinamically generated wheel + run: | + python -m pip install poetry toml + python build/pypi/openbb_platform/nightly.py + + - name: OpenBB Platform - Publish distribution 📦 to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: openbb_platform/dist/ + password: ${{ secrets.TEST_PYPI_API_TOKEN_OBB_PLATFORM_NIGHTLY }} + repository-url: https://test.pypi.org/legacy/ diff --git a/openbb_platform/PUBLISH.md b/build/pypi/openbb_platform/PUBLISH.md similarity index 79% rename from openbb_platform/PUBLISH.md rename to build/pypi/openbb_platform/PUBLISH.md index 7b55cbae1874..f1283a082747 100644 --- a/openbb_platform/PUBLISH.md +++ b/build/pypi/openbb_platform/PUBLISH.md @@ -7,15 +7,15 @@ Publishing checklist: 1. Ensure all unit tests pass: `pytest openbb_platform -m "not integration"` 2. Ensure all integration tests pass: `pytest openbb_platform -m integration` 3. Change the Platform version on: - - `openbb_platform/platform/core/openbb_core/app/constants.py` + - `openbb_platform/platform/core/openbb_core/app/constants.py` - `openbb_platform/README.md` -4. Run the publishing script: `python openbb_platform/dev_utils/dev_publish.py` -5. Update poetry files: `python openbb_platform/dev_utils/poetry_update.py` +4. Run the publishing script: `python build/pypi/openbb_platform/publish.py` +5. Update poetry files: `python build/pypi/openbb_platform/poetry_update.py` 6. Open a PR so that changes are reflected on the main branch 7. If applicable, set the version on `constants.py` to use the `dev` tag again Finally, check if everything works: -1. Install and test the package from Pypi on a clean environment +1. Install and test the package from Pypi on a clean environment. 2. Check if all the `pyproject.toml` files are correct, including the `openbb_platform` one. 3. Double check if there is any new extension or provider that needs to be added to [integration tests GitHub Action workflow](/.github/workflows/platform-api-integration-test.yml). diff --git a/build/pypi/openbb_platform/nightly.py b/build/pypi/openbb_platform/nightly.py new file mode 100644 index 000000000000..1b1af491ec04 --- /dev/null +++ b/build/pypi/openbb_platform/nightly.py @@ -0,0 +1,83 @@ +import subprocess +import sys +from pathlib import Path +from typing import Any, Dict + +import toml + +PLATFORM_PATH = Path(__file__).parent.parent.parent.parent.resolve() / "openbb_platform" +PYPROJECT = PLATFORM_PATH / "pyproject.toml" + +SUB_PACKAGES = {} +DEPENDENCIES: Dict[str, Any] = {} +PLUGINS = {"openbb_core_extension": {}, "openbb_provider_extension": {}} +CMD = [sys.executable, "-m", "poetry", "build"] + + +PYPROJECT_TOML = toml.load(PYPROJECT) +POETRY_DICT: Dict[str, dict] = PYPROJECT_TOML["tool"]["poetry"] +POETRY_DICT.pop("extras", None) + + +original_pyproject = PYPROJECT.read_text() + + +def gather_metadata(sub_path: str): + """Gather metadata from the pyproject.toml files. + + Parameters + ---------- + sub_path : str + The path to the sub packages. + """ + for path in PLATFORM_PATH.rglob(f"{sub_path}/**/pyproject.toml"): + pyproject_toml = toml.load(path) + poetry_dict: Dict[str, dict] = pyproject_toml["tool"]["poetry"] + package_name = poetry_dict["packages"][0]["include"] + + for extension in list(PLUGINS.keys()): + PLUGINS[extension].update(poetry_dict.get("plugins", {}).get(extension, {})) + + DEPENDENCIES.update(pyproject_toml["tool"]["poetry"]["dependencies"]) + SUB_PACKAGES[package_name] = path.relative_to(PLATFORM_PATH).parent.as_posix() + + +def build(): + """Build the Platform package.""" + for sub_path in ["platform", "providers", "extensions"]: + gather_metadata(sub_path) + + # need to pop these from the dependencies + DEPENDENCIES.pop("openbb-core", None) + DEPENDENCIES.pop("openbb-provider", None) + + # add the sub packages + for package_name, path in SUB_PACKAGES.items(): + POETRY_DICT["packages"].append({"include": package_name, "from": f"./{path}"}) + + # add the plugins extensions + for extension, plugins in PLUGINS.items(): + POETRY_DICT.setdefault("plugins", {}).setdefault(extension, {}).update(plugins) + + # update the dependencies and platform poetry dict + POETRY_DICT["dependencies"] = DEPENDENCIES + PYPROJECT_TOML["tool"]["poetry"] = POETRY_DICT + + temp_pyproject = toml.dumps(PYPROJECT_TOML) + + try: + with open(PYPROJECT, "w", encoding="utf-8", newline="\n") as f: + f.write(temp_pyproject) + + subprocess.run(CMD, cwd=PLATFORM_PATH, check=True) # noqa: S603,PLW1510 + except (Exception, KeyboardInterrupt) as e: + print(e) # noqa: T201 + print("Restoring pyproject.toml") # noqa: T201 + + # we restore the original pyproject.toml + with open(PYPROJECT, "w", encoding="utf-8", newline="\n") as f: + f.write(original_pyproject) + + +if __name__ == "__main__": + build() diff --git a/openbb_platform/poetry_update.py b/build/pypi/openbb_platform/poetry_update.py similarity index 58% rename from openbb_platform/poetry_update.py rename to build/pypi/openbb_platform/poetry_update.py index a03230c2f9f9..fc3603ba2725 100644 --- a/openbb_platform/poetry_update.py +++ b/build/pypi/openbb_platform/poetry_update.py @@ -1,27 +1,27 @@ -import os +import subprocess import sys from pathlib import Path from typing import List, Optional +PLATFORM_PATH = Path(__file__).parent.parent.parent.parent.resolve() / "openbb_platform" def update_lock_file(packages: Optional[List[str]] = None) -> None: """Update the poetry lock files in all the packages or in the given packages.""" - for path in Path.cwd().rglob("pyproject.toml"): + for path in PLATFORM_PATH.rglob("pyproject.toml"): if packages and path.parent.name not in packages: print(f"Skipping {path.parent.name}") # noqa: T201 continue print(f"\nUpdating lock file in {path.parent.name}") # noqa: T201 - os.chdir(path.parent) - os.system("poetry lock --no-update") # noqa: S605, S607 + CMD = [sys.executable, "-m", "poetry", "lock", "--no-update"] + subprocess.run(CMD, cwd=path.parent, check=True) # noqa: S603,PLW1510 - # Fix line endings and encoding lock_file = path.parent / "poetry.lock" - lock_file.write_text( - lock_file.read_text(), - encoding="utf-8", - newline="\n", - ) + lock_text = lock_file.read_text() + + # Fix line endings and encoding + with open(lock_file, "w", encoding="utf-8", newline="\n") as f: + f.write(lock_text) if __name__ == "__main__": diff --git a/build/pypi/openbb_platform/publish.py b/build/pypi/openbb_platform/publish.py new file mode 100644 index 000000000000..7b63194f0819 --- /dev/null +++ b/build/pypi/openbb_platform/publish.py @@ -0,0 +1,39 @@ +import subprocess +import sys +from pathlib import Path + +PLATFORM_PATH = Path(__file__).parent.resolve() / "openbb_platform" + +SUB_PACKAGES = ["platform/provider", "platform/core", "extensions", "providers"] + +CMD = [sys.executable, "-m", "poetry"] +VERSION_BUMP_CMD = ["version", "prerelease"] +PUBLISH_CMD = ["publish", "--build"] + + +def run_cmds(directory: Path): + """Run the commands for publishing""" + print(f"Publishing: {directory.name}") # noqa: T201 + + subprocess.run(CMD + VERSION_BUMP_CMD, cwd=directory, check=True) # noqa: S603 + subprocess.run(CMD + PUBLISH_CMD, cwd=directory, check=True) # noqa: S603 + + +def publish(): + """Publish the Platform to PyPi""" + for sub_path in SUB_PACKAGES: + for path in PLATFORM_PATH.rglob(f"{sub_path}/**/pyproject.toml"): + run_cmds(path.parent) + + # openbb + run_cmds(PLATFORM_PATH) + + +if __name__ == "__main__": + raise Exception( + "If you're ar running this script for the first time," + "ensure you have changed `VERSION` on System Settings " + "before you publish the `openbb-core` package to Pypi." + ) + + publish() diff --git a/build/pypi/build.sh b/build/pypi/openbb_terminal/build.sh similarity index 100% rename from build/pypi/build.sh rename to build/pypi/openbb_terminal/build.sh diff --git a/build/pypi/test_publish.sh b/build/pypi/openbb_terminal/test_publish.sh similarity index 100% rename from build/pypi/test_publish.sh rename to build/pypi/openbb_terminal/test_publish.sh diff --git a/noxfile.py b/noxfile.py index 308ae289d432..2c1c2aed56d6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -12,7 +12,11 @@ def tests(session): session.install("poetry", "toml") session.run( - "python", "./openbb_platform/dev_install.py", "-e", "all", external=True + "python", + "./openbb_platform/dev_install.py", + "-e", + "all", + external=True, ) session.install("pytest") session.install("pytest-cov") diff --git a/openbb_platform/README.md b/openbb_platform/README.md index d3ff51ade17d..25dff271e0c6 100644 --- a/openbb_platform/README.md +++ b/openbb_platform/README.md @@ -6,21 +6,19 @@ | ![OpenBBLogo](https://user-images.githubusercontent.com/25267873/218899768-1f0964b8-326c-4f35-af6f-ea0946ac970b.png) | | Check our website at [openbb.co](www.openbb.co) | - ## OpenBB Platform Overview The OpenBB Platform provides a convenient way to access raw financial data from multiple data providers. The package comes with a ready to use REST API - this allows developers from any language to easily create applications on top of OpenBB Platform. - ## Installation The command below provides access to the core functionalities behind the OpenBB Platform. ```bash -pip install openbb==4.0.0a4 +pip install openbb --pre ``` -> Note that the version `4.0.0a4` is an example and you should replace it with the latest version available. +> The `--pre` flag is required to install the latest alpha version. To install extensions that expand the core functionalities specify the extension name or use `all` to install all. diff --git a/openbb_platform/dev_install.py b/openbb_platform/dev_install.py index b8d6e6ec702a..b49eb1a48076 100644 --- a/openbb_platform/dev_install.py +++ b/openbb_platform/dev_install.py @@ -76,7 +76,9 @@ def install_local(_extras: bool = False): try: # we create a temporary pyproject.toml - PYPROJECT.write_text(TEMP_PYPROJECT) + with open(PYPROJECT, "w", encoding="utf-8", newline="\n") as f: + f.write(TEMP_PYPROJECT) + CMD = [sys.executable, "-m", "poetry"] subprocess.run( # noqa: PLW1510 diff --git a/openbb_platform/dev_publish.py b/openbb_platform/dev_publish.py deleted file mode 100644 index a4f1d3db1969..000000000000 --- a/openbb_platform/dev_publish.py +++ /dev/null @@ -1,59 +0,0 @@ -import os -from pathlib import Path - -repo_dir = Path(__file__).parent.parent - -# core and provider -core_dir = repo_dir / "openbb_platform/platform/core" -provider_dir = repo_dir / "openbb_platform/platform/provider" - -# extensions -extensions_dir = repo_dir / "openbb_platform/extensions" -extensions = [x for x in extensions_dir.iterdir() if x.is_dir()] - -# providers -providers_dir = repo_dir / "openbb_platform/providers" -providers = [x for x in providers_dir.iterdir() if x.is_dir()] - -# openbb -openbb_dir = repo_dir / "openbb_platform" - -VERSION_BUMP_CMD = "poetry version prerelease" -PUBLISH_CMD = "poetry publish --build" - -raise Exception( - "If you're ar running this script for the first time," - "ensure you have changed `VERSION` on System Settings " - "before you publish the `openbb-core` package to Pypi." -) - - -def run_cmds(directory: Path): - print(f"Publishing: {directory.name}") # noqa: T201 - os.chdir(directory) - os.system(VERSION_BUMP_CMD) # noqa: S605 - os.system(PUBLISH_CMD) # noqa: S605 - - -# provider -run_cmds(provider_dir) - -# core -run_cmds(core_dir) - -# extensions -for extension in extensions: - if extension.name in ["__pycache__", "tests"]: - continue - - run_cmds(extension) - -# providers -for provider in providers: - if provider.name in ["__pycache__", "tests"]: - continue - - run_cmds(provider) - -# openbb -run_cmds(openbb_dir) diff --git a/pyproject.toml b/pyproject.toml index 4308fd8315da..0a3a76e26fb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "openbb" +name = "openbb-terminal" version = "3.2.3" description = "Investment Research for Everyone, Anywhere." license = "MIT" diff --git a/website/content/platform/faqs/installation_updates.md b/website/content/platform/faqs/installation_updates.md index 562eae859993..6e5329532f2d 100644 --- a/website/content/platform/faqs/installation_updates.md +++ b/website/content/platform/faqs/installation_updates.md @@ -70,7 +70,7 @@ poetry install -E all The nightly build can be installed with: ```console -pip install openbb-nightly[all] +pip install openbb-terminal-nightly[all] ``` **Note**: This version may not be stable and should not be used in a production setting. diff --git a/website/versioned_docs/version-v3/sdk/faqs/installation_updates.md b/website/versioned_docs/version-v3/sdk/faqs/installation_updates.md index 68651eec43ad..055543c4020f 100644 --- a/website/versioned_docs/version-v3/sdk/faqs/installation_updates.md +++ b/website/versioned_docs/version-v3/sdk/faqs/installation_updates.md @@ -70,7 +70,7 @@ poetry install -E all The nightly build can be installed with: ```console -pip install openbb-nightly[all] +pip install openbb-terminal-nightly[all] ``` **Note**: This version may not be stable and should not be used in a production setting. diff --git a/website/versioned_docs/version-v3/terminal/installation/pypi.md b/website/versioned_docs/version-v3/terminal/installation/pypi.md index efc89b05743a..72efbee6dcbe 100644 --- a/website/versioned_docs/version-v3/terminal/installation/pypi.md +++ b/website/versioned_docs/version-v3/terminal/installation/pypi.md @@ -216,7 +216,7 @@ Make sure to have completed all previous steps. If followed, there will be a vir Install the main package of Openbb SDK with `pip`, a package manager. ```shell -pip install openbb --no-cache-dir +pip install openbb-terminal --no-cache-dir ``` This method provides access to the data aggregation and charting functions of the OpenBB SDK. It does not provide access to the advanced features that are provided by the Portfolio Optimization and Machine Learning toolkits. @@ -224,23 +224,23 @@ This method provides access to the data aggregation and charting functions of th The toolkits can be installed individually with: ```shell -pip install "openbb[optimization]" --no-cache-dir +pip install "openbb-terminal[optimization]" --no-cache-dir ``` and ```shell -pip install "openbb[forecast]" --no-cache-dir +pip install "openbb-terminal[forecast]" --no-cache-dir ``` Install all available toolkits at once with: ```shell -pip install "openbb[all]" --no-cache-dir +pip install "openbb-terminal[all]" --no-cache-dir ``` :::info -`pip install openbb[all]` is not yet compatible with environments such as Google Colab and Kaggle as they come with preinstalled packages that can conflict with the ones used in the OpenBBTerminal and SDK. It may be possible to install without the extra toolkits, but we currently do not officially support this type of installation. We are working on a solution to this problem and will update this section once it is resolved. +`pip install openbb-terminal[all]` is not yet compatible with environments such as Google Colab and Kaggle as they come with preinstalled packages that can conflict with the ones used in the OpenBBTerminal and SDK. It may be possible to install without the extra toolkits, but we currently do not officially support this type of installation. We are working on a solution to this problem and will update this section once it is resolved. ::: ## Verify Installation