#130 rename an option and add some documentations

Signed-off-by: Long Zhang <[email protected]>

docs/getting-started.md

@@ -192,6 +192,32 @@ backend:
 
 Datadog doesn't provide daily costs. Current daily costs are calculated by `monthly costs/number of days in that month`.
 
+## Autoloading Cost Data and Saving Them Into the Database
+
+In order to improve the performance and reduce the number of calls to the cloud providers, by default, for all the integrations, there is a scheduled [task](../plugins/infrawallet-backend/src/service/router.ts#L57) in the InfraWallet backend that queries the cost data from the cloud provider APIs and saves them into the database (tables `cost_items_daily` and `cost_items_monthly`). The default interval for this task is 8 hours which will be made configurable in the future. It is recommended to have a value that is larger or equal to 8 because most of the cloud providers do not update the cost data very frequently. This feature works similarly as caching but it only autloads the costs data without any filters (such as a cost allocation tag filter in AWS). If a query contains such a filter condition, InfraWallet still makes API calls to the cloud and then caches the data in memory isntead of the database.
+
+If you want to disable this feature, for example, when you run a local development environment with a Sqlite3 database, you can add the following configuration to your `app-config.yaml` file:
+
+```yaml
+backend:
+  infraWallet:
+    autoload:
+      enabled: false
+```
+
+If there is an issue about the historical data in the plugin database, you can use the following two APIs to clean up
+the data and reload the data.
+
+```bash
+# for a prod environment, you may need extra headers like an auth token, etc.
+
+# the following call removes the costs data from the database
+curl -X POST -d '{"granularity": "monthly", "provider": "AWS"}' --header 'Content-Type: application/json' http://localhost:7007/api/infrawallet/default/delete_cost_items
+
+# the following call triggers the fetch and save cloud costs task immediately
+curl http://localhost:7007/api/infrawallet/fetch_and_save_costs
+```
+
 ## Integration Filter
 
 When integrating InfraWallet with your billing account, you have the ability to retrieve and display costs for all sub-accounts. However, if you want to limit the visibility of certain accounts, you can apply filters. Below is an example of how to configure this for AWS:

plugins/infrawallet-backend/config.d.ts

@@ -1,7 +1,10 @@
 export interface Config {
   backend: {
     infraWallet: {
-      prefetchCostData?: boolean; // true to enable prefetching cost data and saving them into plugin db
+      // configurations about prefetching cost data and saving them into plugin db
+      autoload?: {
+        enabled?: boolean;
+      };
       integrations: {
         azure?: {
           name: string;

plugins/infrawallet-backend/src/cost-clients/InfraWalletClient.ts

@@ -255,7 +255,7 @@ export abstract class InfraWalletClient {
   }
 
   async getCostReports(query: CostQuery): Promise<ClientResponse> {
-    const prefetchCostData = this.config.getOptionalBoolean('backend.infraWallet.prefetchCostData') ?? true;
+    const autoloadCostData = this.config.getOptionalBoolean('backend.infraWallet.autoload.enabled') ?? true;
     const integrationConfigs = this.config.getOptionalConfigArray(
       `backend.infraWallet.integrations.${this.provider.toLowerCase()}`,
     );
@@ -266,8 +266,8 @@ export abstract class InfraWalletClient {
     const results: Report[] = [];
     const errors: CloudProviderError[] = [];
 
-    // if prefetchCostData enabled, for a query without any tags or groups, we get the results from the plugin database
-    if (query.tags === '()' && query.groups === '' && prefetchCostData) {
+    // if autoloadCostData enabled, for a query without any tags or groups, we get the results from the plugin database
+    if (query.tags === '()' && query.groups === '' && autoloadCostData) {
       const reportsFromDatabase = await this.getCostReportsFromDatabase(query);
       reportsFromDatabase.forEach(report => {
         results.push(report);

plugins/infrawallet-backend/src/service/router.ts

@@ -45,21 +45,21 @@ export async function createRouter(options: RouterOptions): Promise<express.Rout
   // do database migrations here to support the legacy backend system
   await setUpDatabase(database);
 
-  const prefetchCostData = config.getOptionalBoolean('backend.infraWallet.prefetchCostData') ?? true;
+  const autoloadCostData = config.getOptionalBoolean('backend.infraWallet.autoload.enabled') ?? true;
 
-  if (prefetchCostData) {
+  if (autoloadCostData) {
     // put scheduler here for now to support legacy backends
     await scheduler.scheduleTask({
       frequency: { cron: '0 */8 * * *' }, // every 8 hours
       timeout: { hours: 1 },
-      id: 'infrawallet-fetch-and-save-costs',
+      id: 'infrawallet-autoload-costs',
       fn: async () => {
         await fetchAndSaveCosts(options);
       },
     });
     // trigger this task when the plugin starts up if the task is not running
     try {
-      scheduler.triggerTask('infrawallet-fetch-and-save-costs');
+      scheduler.triggerTask('infrawallet-autoload-costs');
     } catch (e) {
       logger.error(e);
     }