diff --git a/docs/getting-started.md b/docs/getting-started.md index 2ff6785..1d9e65c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -192,6 +192,32 @@ backend: Datadog doesn't provide daily costs. Current daily costs are calculated by `monthly costs/number of days in that month`. +## Autoloading Cost Data and Saving Them Into the Database + +In order to improve the performance and reduce the number of calls to the cloud providers, by default, for all the integrations, there is a scheduled [task](../plugins/infrawallet-backend/src/service/router.ts#L57) in the InfraWallet backend that queries the cost data from the cloud provider APIs and saves them into the database (tables `cost_items_daily` and `cost_items_monthly`). The default interval for this task is 8 hours which will be made configurable in the future. It is recommended to have a value that is larger or equal to 8 because most of the cloud providers do not update the cost data very frequently. This feature works similarly as caching but it only autloads the costs data without any filters (such as a cost allocation tag filter in AWS). If a query contains such a filter condition, InfraWallet still makes API calls to the cloud and then caches the data in memory isntead of the database. + +If you want to disable this feature, for example, when you run a local development environment with a Sqlite3 database, you can add the following configuration to your `app-config.yaml` file: + +```yaml +backend: + infraWallet: + autoload: + enabled: false +``` + +If there is an issue about the historical data in the plugin database, you can use the following two APIs to clean up +the data and reload the data. + +```bash +# for a prod environment, you may need extra headers like an auth token, etc. + +# the following call removes the costs data from the database +curl -X POST -d '{"granularity": "monthly", "provider": "AWS"}' --header 'Content-Type: application/json' http://localhost:7007/api/infrawallet/default/delete_cost_items + +# the following call triggers the fetch and save cloud costs task immediately +curl http://localhost:7007/api/infrawallet/fetch_and_save_costs +``` + ## Integration Filter When integrating InfraWallet with your billing account, you have the ability to retrieve and display costs for all sub-accounts. However, if you want to limit the visibility of certain accounts, you can apply filters. Below is an example of how to configure this for AWS: diff --git a/plugins/infrawallet-backend/config.d.ts b/plugins/infrawallet-backend/config.d.ts index e8f7d09..1118522 100644 --- a/plugins/infrawallet-backend/config.d.ts +++ b/plugins/infrawallet-backend/config.d.ts @@ -1,7 +1,10 @@ export interface Config { backend: { infraWallet: { - prefetchCostData?: boolean; // true to enable prefetching cost data and saving them into plugin db + // configurations about prefetching cost data and saving them into plugin db + autoload?: { + enabled?: boolean; + }; integrations: { azure?: { name: string; diff --git a/plugins/infrawallet-backend/src/cost-clients/InfraWalletClient.ts b/plugins/infrawallet-backend/src/cost-clients/InfraWalletClient.ts index 398223c..686bbd5 100644 --- a/plugins/infrawallet-backend/src/cost-clients/InfraWalletClient.ts +++ b/plugins/infrawallet-backend/src/cost-clients/InfraWalletClient.ts @@ -255,7 +255,7 @@ export abstract class InfraWalletClient { } async getCostReports(query: CostQuery): Promise { - const prefetchCostData = this.config.getOptionalBoolean('backend.infraWallet.prefetchCostData') ?? true; + const autoloadCostData = this.config.getOptionalBoolean('backend.infraWallet.autoload.enabled') ?? true; const integrationConfigs = this.config.getOptionalConfigArray( `backend.infraWallet.integrations.${this.provider.toLowerCase()}`, ); @@ -266,8 +266,8 @@ export abstract class InfraWalletClient { const results: Report[] = []; const errors: CloudProviderError[] = []; - // if prefetchCostData enabled, for a query without any tags or groups, we get the results from the plugin database - if (query.tags === '()' && query.groups === '' && prefetchCostData) { + // if autoloadCostData enabled, for a query without any tags or groups, we get the results from the plugin database + if (query.tags === '()' && query.groups === '' && autoloadCostData) { const reportsFromDatabase = await this.getCostReportsFromDatabase(query); reportsFromDatabase.forEach(report => { results.push(report); diff --git a/plugins/infrawallet-backend/src/service/router.ts b/plugins/infrawallet-backend/src/service/router.ts index c61d560..79e885c 100644 --- a/plugins/infrawallet-backend/src/service/router.ts +++ b/plugins/infrawallet-backend/src/service/router.ts @@ -45,21 +45,21 @@ export async function createRouter(options: RouterOptions): Promise { await fetchAndSaveCosts(options); }, }); // trigger this task when the plugin starts up if the task is not running try { - scheduler.triggerTask('infrawallet-fetch-and-save-costs'); + scheduler.triggerTask('infrawallet-autoload-costs'); } catch (e) { logger.error(e); }