Merge provider and core into core (#5774)

* small changes - moving provider inside the core

* uptade readme

* lint

* remove old provider readme

* leftovers

* refactor website examples

* ruff

* fix nox

* relock

* fix fails_to_deliver headers

* fix for tests that have renamed endpoints

* fix discovery

* removed test for renamed endpoint

* fix broken syntax - test_search_financial_attributes

* fix cash/income/balance

* fix derivatives tests

* fix etf tests

* fix news tests

* fix regulators

* black

* wrong test param

* sort equity gainers/losers by percent

* platform markdown generation import

* Hotfix/fix int tests core merge (#5779)

* removing tests on old endpoint

* removing visualization on old endpoint

* fixing equity tests

* fixing examples

* fix news tests

* fix index test

* example

* add comments

* example

---------

Co-authored-by: Diogo Sousa <[email protected]>

---------

Co-authored-by: montezdesousa <[email protected]>
Co-authored-by: Diogo Sousa <[email protected]>
Co-authored-by: Danglewood <[email protected]>
Co-authored-by: Pratyush Shukla <[email protected]>

build/pypi/openbb_platform/PUBLISH.md

@@ -10,11 +10,10 @@ Publishing checklist:
 
     Consider using the `--dry-run` flag to check if everything is correct before publishing.
     Also, it might be a good idea to run the script in batches to ensure that the packages are published correctly and the dependencies pick the correct versions.
-    Example, the `openbb-core` needs to pick the latest `openbb-provider` version and extension packages need to pick the latest `openbb-core` version.
+    Example, the extension packages need to pick the latest `openbb-core` version.
     Suggested batch order:
     1. Batch 1. Core
-       1. `openbb-provider`
-       2. `openbb-core`
+       1. `openbb-core`
     2. Batch 2. Extensions
        1. `openbb-charting`
        2. `openbb-crypto`

build/pypi/openbb_platform/nightly.py

@@ -49,7 +49,6 @@ def build():
 
     # need to pop these from the dependencies
     DEPENDENCIES.pop("openbb-core", None)
-    DEPENDENCIES.pop("openbb-provider", None)
 
     # add the sub packages
     for package_name, path in SUB_PACKAGES.items():

build/pypi/openbb_platform/publish.py

@@ -10,7 +10,6 @@
 EXTENSION_PACKAGES = ["extensions", "providers"]
 
 CMD = [sys.executable, "-m", "poetry"]
-CORE_DEPENDENCIES_UPDATE_CMD = ["add", "openbb-provider=latest"]
 EXTENSION_DEPENDENCIES_UPDATE_CMD = ["add", "openbb-core=latest"]
 VERSION_BUMP_CMD = ["version", "prerelease"]
 PUBLISH_CMD = ["publish", "--build"]
@@ -40,15 +39,6 @@ def parse_args():
     return parser.parse_args()
 
 
-def update_core_dependencies(path: Path):
-    """Update the core dependencies"""
-    subprocess.run(
-        CMD + CORE_DEPENDENCIES_UPDATE_CMD,  # noqa: S603
-        cwd=path.parent,
-        check=True,
-    )
-
-
 def update_extension_dependencies(path: Path):
     """Update the extension dependencies"""
     subprocess.run(
@@ -74,17 +64,11 @@ def publish(dry_run: bool = False, core: bool = False, extensions: bool = False)
         package_paths.extend(EXTENSION_PACKAGES)
 
     for sub_path in package_paths:
-        is_core = sub_path in CORE_PACKAGES
         is_extension = sub_path in EXTENSION_PACKAGES
 
         for path in PLATFORM_PATH.rglob(f"{sub_path}/**/pyproject.toml"):
             try:
                 # Update dependencies
-                if is_core:
-                    # if it's the provider package no need to update the dependencies
-                    if "provider" in str(path.parent):
-                        continue
-                    update_core_dependencies(path)
                 if is_extension:
                     update_extension_dependencies(path)
                 # Bump version

noxfile.py

@@ -2,7 +2,7 @@
 
 test_locations = [
     "openbb_platform/tests",
-    "openbb_platform/platform",
+    "openbb_platform/core",
     "openbb_platform/providers",
     "openbb_platform/extensions",
 ]

openbb_platform/CONTRIBUTING.md

@@ -119,7 +119,7 @@ Before moving forward, please take a look at the high-level view of the OpenBB P
 
 The Standardization Framework is a set of tools and guidelines that enable the user to query and obtain data in a consistent way across multiple providers.
 
-Each data model should inherit from a [standard data](platform/provider/openbb_provider/standard_models) model that is already defined inside the OpenBB Platform. All standard models are created and maintained by the OpenBB team.
+Each data model should inherit from a [standard data](platform/provider/openbb_core/provider/standard_models) model that is already defined inside the OpenBB Platform. All standard models are created and maintained by the OpenBB team.
 
 Usage of these models will unlock a set of perks that are only available to standardized data, namely:
 
@@ -129,9 +129,9 @@ Usage of these models will unlock a set of perks that are only available to stan
 - Can expect consistent data types and validation.
 - Will work seamlessly with other providers that use the same standard model.
 
-The standard models are defined under the `/OpenBBTerminal/openbb_platform/platform/core/provider/openbb_provider/standard_models/` directory.
+The standard models are defined under the `/OpenBBTerminal/openbb_platform/platform/core/provider/openbb_core/provider/standard_models/` directory.
 
-They define the [`QueryParams`](platform/provider/openbb_provider/abstract/query_params.py) and [`Data`](platform/provider/openbb_provider/abstract/data.py) models, which are used to query and output data. They are pydantic and you can leverage all the pydantic features such as validators.
+They define the [`QueryParams`](platform/provider/openbb_core/provider/abstract/query_params.py) and [`Data`](platform/provider/openbb_core/provider/abstract/data.py) models, which are used to query and output data. They are pydantic and you can leverage all the pydantic features such as validators.
 
 ##### Standardization Caveats
 
@@ -287,7 +287,7 @@ The high level steps are:
 
 ### How to add a new data point?
 
-In this section, we'll be adding a new data point to the OpenBB Platform. We will add a new provider with an existing [standard data](platform/provider/openbb_provider/standard_models) model.
+In this section, we'll be adding a new data point to the OpenBB Platform. We will add a new provider with an existing [standard data](platform/provider/openbb_core/provider/standard_models) model.
 
 #### Identify which type of data you want to add
 
@@ -300,7 +300,7 @@ Each router is categorized under different extensions (equity, currency, crypto,
 
 Given the fact that there's already an endpoint for OHLCV stock data, we can check if the standard exists.
 
-In this case, it's `EquityHistorical` which can be found in `/OpenBBTerminal/openbb_platform/platform/core/provider/openbb_provider/standard_models/equity_historical`.
+In this case, it's `EquityHistorical` which can be found in `/OpenBBTerminal/openbb_platform/platform/core/provider/openbb_core/provider/standard_models/equity_historical`.
 
 If the standard model doesn't exist:
 
@@ -415,7 +415,7 @@ In order to make the new provider visible to the OpenBB Platform, you'll need to
 
 ```python
 """<Provider Name> Provider module."""
-from openbb_provider.abstract.provider import Provider
+from openbb_core.provider.abstract.provider import Provider
 
 from openbb_<provider_name>.models.equity_historical import <ProviderName>EquityHistoricalFetcher
 
@@ -444,8 +444,8 @@ Saying that, we highly recommend following the standardization framework, as it
 
 When standardizing, all data is defined using two different pydantic models:
 
-1. Define the [query parameters](platform/provider/openbb_provider/abstract/query_params.py) model.
-2. Define the resulting [data schema](platform/provider/openbb_provider/abstract/data.py) model.
+1. Define the [query parameters](platform/provider/openbb_core/provider/abstract/query_params.py) model.
+2. Define the resulting [data schema](platform/provider/openbb_core/provider/abstract/data.py) model.
 
 > The models can be entirely custom, or inherit from the OpenBB standardized models.
 > They enforce a safe and consistent data structure, validation and type checking.
@@ -458,9 +458,9 @@ After you've defined both models, you'll need to define a `Fetcher` class which
 2. `extract_data` - makes the request to the API endpoint and returns the raw data.
 3. `transform_data` - transforms the raw data into the defined data model.
 
-> Note that the `Fetcher` should inherit from the [`Fetcher`](platform/provider/openbb_provider/abstract/fetcher.py) class, which is a generic class that receives the query parameters and the data model as type parameters.
+> Note that the `Fetcher` should inherit from the [`Fetcher`](platform/provider/openbb_core/provider/abstract/fetcher.py) class, which is a generic class that receives the query parameters and the data model as type parameters.
 
-After finalizing your models, you need to make them visible to the Openbb Platform. This is done by adding the `Fetcher` to the `__init__.py` file of the `<your_package_name>/<your_module_name>` folder as part of the [`Provider`](platform/provider/openbb_provider/abstract/provider.py).
+After finalizing your models, you need to make them visible to the Openbb Platform. This is done by adding the `Fetcher` to the `__init__.py` file of the `<your_package_name>/<your_module_name>` folder as part of the [`Provider`](platform/provider/openbb_core/provider/abstract/provider.py).
 
 Any command, that uses the `Fetcher` class you've just defined, will be calling the `transform_query`, `extract_data` and `transform_data` methods under the hood in order to get the data and output it do the end user.
 
@@ -513,13 +513,13 @@ You only need to change the `model` parameter to the name of the `Fetcher` dicti
 ```python
 
 # The `Data` class
-from openbb_provider.abstract.data import Data
+from openbb_core.provider.abstract.data import Data
 
 # The `QueryParams` class
-from openbb_provider.abstract.query_params import QueryParams
+from openbb_core.provider.abstract.query_params import QueryParams
 
 # The `Fetcher` class
-from openbb_provider.abstract.fetcher import Fetcher
+from openbb_core.provider.abstract.fetcher import Fetcher
 
 # The `OBBject` class
 from openbb_core.app.model.obbject import OBBject
@@ -534,14 +534,14 @@ from openbb_core.app.router import Router
 The TET pattern is a pattern that we use to build the `Fetcher` classes. It stands for **Transform, Extract, Transform**.
 As the OpenBB Platform has its own standardization framework and the data fetcher are a very important part of it, we need to ensure that the data is transformed and extracted in a consistent way, to help us do that, we came up with the **TET** pattern, which helps us build and ship faster as we have a clear structure on how to build the `Fetcher` classes.
 
-1. Transform - `transform_query(params: Dict[str, Any])`: transforms the query parameters. Given a `params` dictionary this method should return the transformed query parameters as a [`QueryParams`](openbb_platform/platform/provider/openbb_provider/abstract/query_params.py) child so that we can leverage the pydantic model schemas and validation into the next step. This might also be the place do perform some transformations on any given parameter, i.e., if you want to transform an empty date into a `datetime.now().date()`.
+1. Transform - `transform_query(params: Dict[str, Any])`: transforms the query parameters. Given a `params` dictionary this method should return the transformed query parameters as a [`QueryParams`](openbb_platform/platform/provider/openbb_core/provider/abstract/query_params.py) child so that we can leverage the pydantic model schemas and validation into the next step. This might also be the place do perform some transformations on any given parameter, i.e., if you want to transform an empty date into a `datetime.now().date()`.
 2. Extract - `extract_data(query: ExampleQueryParams,credentials: Optional[Dict[str, str]],**kwargs: Any,) -> Dict`: makes the request to the API endpoint and returns the raw data. Given the transformed query parameters, the credentials and any other extra arguments, this method should return the raw data as a dictionary.
-3. Transform - `transform_data(query: ExampleQueryParams, data: Dict, **kwargs: Any) -> List[ExampleHistoricalData]`: transforms the raw data into the defined data model. Given the transformed query parameters (might be useful for some filtering), the raw data and any other extra arguments, this method should return the transformed data as a list of [`Data`](openbb_platform/platform/provider/openbb_provider/abstract/data.py) children.
+3. Transform - `transform_data(query: ExampleQueryParams, data: Dict, **kwargs: Any) -> List[ExampleHistoricalData]`: transforms the raw data into the defined data model. Given the transformed query parameters (might be useful for some filtering), the raw data and any other extra arguments, this method should return the transformed data as a list of [`Data`](openbb_platform/platform/provider/openbb_core/provider/abstract/data.py) children.
 
 #### Data processing commands
 
 The data processing commands are commands that are used to process the data that may or may not come from the OpenBB Platform.
-In order to create a data processing framework general enough to be used by any extension, we've created a special abstract class called [`Data`](/openbb_platform/platform/provider/openbb_provider/abstract/data.py) which **all** standardized (and consequently its child classes) will inherit from.
+In order to create a data processing framework general enough to be used by any extension, we've created a special abstract class called [`Data`](/openbb_platform/platform/provider/openbb_core/provider/abstract/data.py) which **all** standardized (and consequently its child classes) will inherit from.
 
 Why is this important?
 So that we can ensure that all `OBBject.results` will share a common ground on which we can apply out-of-the-box data processing commands, such as the `ta`, `qa` or the `econometrics` menus.
@@ -588,7 +588,7 @@ Not at all! Consider the following example:
 
 ```python
 
->>> from openbb_provider.abstract.data import Data
+>>> from openbb_core.provider.abstract.data import Data
 >>> my_data_item_1 = {"open": 1, "high": 2, "low": 3, "close": 4, "volume": 5, "date": "2020-01-01"}
 >>> my_data_item_1_as_data = Data.model_validate(my_data_item_1)
 >>> my_data_item_1_as_data

openbb_platform/platform/core/README.md → openbb_platform/core/README.md

@@ -2,6 +2,9 @@
 
 - [THIS README IS A WORK IN PROGRESS](#this-readme-is-a-work-in-progress)
   - [1. Introduction](#1-introduction)
+    - [1.1 Data Providers](#11-data-providers)
+    - [1.2 Data Standardization](#12-data-standardization)
+    - [1.3 Key Elements](#13-key-elements)
   - [2. How to install?](#2-how-to-install)
     - [Git clone](#git-clone)
     - [Install](#install)
@@ -37,6 +40,43 @@
 
 This directory contains the OpenBB Platform's core functionality. It allows you to create an [extension](../../extensions/README.md) or a [provider](../../providers/README.md) that will be automatically turned into REST API endpoint and allow sharing data between commands.
 
+### 1.1 Data Providers
+
+OpenBB aims to give a coherent access to financial data providers by introducing standardization procedures.
+
+### 1.2 Data Standardization
+
+It's like teaching everyone to speak the same language with their data so we can understand and compare it easily.
+
+**Think of it as a part of**:
+
+Data normalization, a bigger way of organizing data.
+
+**What We Do**:
+
+- **Match Column Names and Formats**: Like making sure everyone calls a "closing price" the same thing.
+
+- **For instance**: Some might say "Close", others might say "c". We make sure everyone uses one term.
+**Unify Date and Time Styles**: Like having everyone use the same calendar format.
+
+**Example**: Whether it's "YYYY-MM-DD" or "MM-DD-YYYY", we pick one style for everyone.
+
+**What We Don’t Do**:
+
+- **Cleaning Data**: We don't act like data detectives and remove mistakes from what providers give us.
+
+- **Change & Combine Data**: Tweaking data or mixing data from different places.
+
+### 1.3 Key Elements
+
+- **QueryParams** : The input model for a particular query. To load equity market data, we would have `EquityPriceQueryParams`, which would have fields like `symbol`, `start_date`, and `end_date`. You can find the standard query params inside the `standard_models` directory.
+- **Data** : The output model of a particular query. Equity market data would be `EquityPriceData` and have fields such as `Open`, `High`, `Low`, `Close`, and `Volume`. You can find the standard data models inside the `standard_models` directory.
+- **Fetcher** : Class containing a set of methods to receive query parameters, extract data and transform it. This class is responsible for implementing the standardization procedures.
+- **Provider** : Entry point class for each provider extension. Contains information about the provider, it's required credentials and available fetchers.
+- **RegistryLoader** : Loads the registry with the installed provider extensions.
+- **Registry** : Maintains a registry of provider extensions installed.
+- **RegistryMap** : Stores the complete characterization of each provider. It centralizes information like required credentials, standardized and extra query parameters/data by provider.
+- **QueryExecutor** : Executes a given query, routing it to the respective provider and fetcher.
 
 ## 2. How to install?
 
@@ -83,9 +123,9 @@ def some_command(
     pass
 ```
 
-If your command only makes use of a standard model defined inside `openbb_provider/standard_models` directory, there is no need to repeat its structure in the parameters. Just pass the model name as an argument.
+If your command only makes use of a standard model defined inside `openbb_core/provider/standard_models` directory, there is no need to repeat its structure in the parameters. Just pass the model name as an argument.
 
-This is an example how we do it for `equity.price.historical` which only depends on `EquityHistorical` model defined in `openbb-provider`:
+This is an example how we do it for `equity.price.historical` which only depends on `EquityHistorical` model defined in `openbb-core.provider.standard_models`:
 
 ```python
 @router.command(model="EquityHistorical")

openbb_platform/platform/core/openbb_core/app/model/metadata.py → openbb_platform/core/openbb_core/app/model/metadata.py

@@ -4,9 +4,10 @@
 
 import numpy as np
 import pandas as pd
-from openbb_provider.abstract.data import Data
 from pydantic import BaseModel, Field, field_validator
 
+from openbb_core.provider.abstract.data import Data
+
 
 class Metadata(BaseModel):
     arguments: Dict[str, Any] = Field(

openbb_platform/platform/core/openbb_core/app/provider_interface.py → openbb_platform/core/openbb_core/app/provider_interface.py

@@ -4,9 +4,6 @@
 from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union
 
 from fastapi import Query
-from openbb_provider.query_executor import QueryExecutor
-from openbb_provider.registry_map import MapType, RegistryMap
-from openbb_provider.utils.helpers import to_snake_case
 from pydantic import (
     AliasChoices,
     BaseModel,
@@ -17,6 +14,9 @@
 from pydantic.fields import FieldInfo
 
 from openbb_core.app.model.abstract.singleton import SingletonMeta
+from openbb_core.provider.query_executor import QueryExecutor
+from openbb_core.provider.registry_map import MapType, RegistryMap
+from openbb_core.provider.utils.helpers import to_snake_case
 
 TupleFieldType = Tuple[str, type, Any]
 
@@ -58,8 +58,6 @@ class ProviderChoices:
 class ProviderInterface(metaclass=SingletonMeta):
     """Provider interface class.
 
-    Provides access to 'openbb_provider' package information.
-
     Properties
     ----------
     map : MapType