1. Regression test AWS source creation with PATCH POST http://localhost:3000/api/v1.0/sources/ {"source_type_id": "1", "name": "AWS Source"} Response { "id": 1, "name": "AWS Source", "uid": "fde0841f-92fb-4437-bda1-60feeabc4a98", "created_at": "2021-01-22T14:27:03.489Z", "updated_at": "2021-01-22T14:27:03.489Z", "source_type_id": 1, "version": null, "availability_status": null, "imported": null, "source_ref": null, "last_checked_at": null, "last_available_at": null, "tenant": "10001" } POST http://localhost:3000/api/v1.0/applications/ {"source_id": "1", "application_type_id": "2"} Response { "id": 1, "source_id": 1, "application_type_id": 2, "created_at": "2021-01-22T14:27:40.292Z", "updated_at": "2021-01-22T14:27:40.292Z", "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "extra": {}, "tenant": "10001" } POST http://localhost:3000/api/v1.0/authentications/ { "authtype": "arn", "name": "", "password": "arn:aws:iam::999999999999:role/CostManagement", "status": "valid", "status_details": "Details Here", "username": "username", "resource_type": "Application", "resource_id": "1" } Response { "id": 1, "resource_type": "Application", "resource_id": 1, "name": "", "authtype": "arn", "username": "username", "password": "arn:aws:iam::999999999999:role/CostManagement", "status": "valid", "status_details": "Details Here", "extra": null, "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "source_id": 1, "tenant": "10001" } PATCH http://localhost:8000/api/cost-management/v1/sources/1/ {"billing_source": {"bucket": "cost-usage-bucket"}} { "id": 1, "uuid": "fde0841f-92fb-4437-bda1-60feeabc4a98", "name": "AWS Source", "source_type": "AWS", "authentication": { "credentials": { "role_arn": "arn:aws:iam::999999999999:role/CostManagement" } }, "billing_source": { "data_source": { "bucket": "cost-usage-bucket" } } } Source Successful GET /api/cost-management/v1/sources/1/ HTTP 200 OK Allow: GET, HEAD, PATCH, DELETE Cache-Control: max-age=0, no-cache, no-store, must-revalidate, private Content-Type: application/json Expires: Fri, 22 Jan 2021 14:29:46 GMT Vary: Accept { "id": 1, "uuid": "fde0841f-92fb-4437-bda1-60feeabc4a98", "name": "AWS Source", "source_type": "AWS", "authentication": { "credentials": { "role_arn": "arn:aws:iam::999999999999:role/CostManagement" } }, "billing_source": { "data_source": { "bucket": "cost-usage-bucket" } }, "provider_linked": true, "active": true, "current_month_data": true, "infrastructure": "Unknown", "cost_models": [] } sources_api_development=# select id, availability_status from applications where id=1; id | availability_status ----+--------------------- 1 | available (1 row) sources_api_development=# 2. Regression test Azure source creation with PATCH POST http://localhost:3000/api/v1.0/sources/ {"source_type_id": "3", "name": "Azure Source"} Response: { "id": 2, "name": "Azure Source", "uid": "c28370bc-0aea-4350-9119-26108909fedd", "created_at": "2021-01-22T14:31:00.513Z", "updated_at": "2021-01-22T14:31:00.513Z", "source_type_id": 3, "version": null, "availability_status": null, "imported": null, "source_ref": null, "last_checked_at": null, "last_available_at": null, "tenant": "10001" } POST http://localhost:3000/api/v1.0/applications/ {"source_id": "2", "application_type_id": "2"} Response: { "id": 2, "source_id": 2, "application_type_id": 2, "created_at": "2021-01-22T14:31:27.157Z", "updated_at": "2021-01-22T14:31:27.157Z", "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "extra": {}, "tenant": "10001" } POST http://localhost:3000/api/v1.0/authentications/ { "authtype": "tenant_id_client_id_client_secret", "name": "", "password": "redacted-secret", "status": "valid", "status_details": "Details Here", "username": "redacted-client-id", "resource_type": "Application", "extra": {"azure": {"tenant_id": "redacted-tenant-id"}}, "resource_id": "2" } Response: { "id": 2, "resource_type": "Application", "resource_id": 2, "name": "", "authtype": "tenant_id_client_id_client_secret", "username": "redacted-client-id", "password": "redacted-secret", "status": "valid", "status_details": "Details Here", "extra": { "azure": { "tenant_id": "redacted-tenant-id" } }, "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "source_id": 2, "tenant": "10001" } PATCH http://localhost:8000/api/cost-management/v1/sources/2/ {"billing_source": {"data_source": {"resource_group": "redacted-rg", "storage_account": "redacted-sa"}}} Response: { "id": 2, "uuid": "c28370bc-0aea-4350-9119-26108909fedd", "name": "Azure Source", "source_type": "Azure", "authentication": { "credentials": { "client_id": "redacted-client-id", "tenant_id": "redacted-tenant-id", "client_secret": "redacted-secret" } }, "billing_source": { "data_source": { "resource_group": "redacted-rg", "storage_account": "redacted-sa" } } } PATCH http://localhost:8000/api/cost-management/v1/sources/2/ {"authentication": {"credentials": {"subscription_id": "redacted-sub-id"}}} Response: { "id": 2, "uuid": "c28370bc-0aea-4350-9119-26108909fedd", "name": "Azure Source", "source_type": "Azure", "authentication": { "credentials": { "client_id": "redacted-client-id", "tenant_id": "redacted-tenant-id", "client_secret": "redacted-secret", "subscription_id": "redacted-sub-id" } }, "billing_source": { "data_source": { "resource_group": "redacted-rg", "storage_account": "redacted-sa" } } } Source creation successful: GET /api/cost-management/v1/sources/2/ HTTP 200 OK Allow: GET, HEAD, PATCH, DELETE Cache-Control: max-age=0, no-cache, no-store, must-revalidate, private Content-Type: application/json Expires: Fri, 22 Jan 2021 14:33:36 GMT Vary: Accept { "id": 2, "uuid": "c28370bc-0aea-4350-9119-26108909fedd", "name": "Azure Source", "source_type": "Azure", "authentication": { "credentials": { "client_id": "redacted-client-id", "tenant_id": "redacted-tenant-id", "subscription_id": "redacted-sub-id" } }, "billing_source": { "data_source": { "resource_group": "redacted-rg", "storage_account": "redacted-sa" } }, "provider_linked": true, "active": true, "current_month_data": true, "infrastructure": "Unknown", "cost_models": [] } sources_api_development=# select id, availability_status from applications where id=2; id | availability_status ----+--------------------- 2 | available (1 row) sources_api_development=# 3. Add AWS source with Application-extra flow Note: Removed Azure and AWS source before doing this so koku would accept it POST http://localhost:3000/api/v1.0/sources/ {"source_type_id": "1", "name": "AWS Source App-Extra"} Response: { "id": 3, "name": "AWS Source App-Extra", "uid": "f227d4b1-6b8f-42bd-9d7c-cd51ac55dd86", "created_at": "2021-01-22T14:35:30.924Z", "updated_at": "2021-01-22T14:35:30.924Z", "source_type_id": 1, "version": null, "availability_status": null, "imported": null, "source_ref": null, "last_checked_at": null, "last_available_at": null, "tenant": "10001" } POST http://localhost:3000/api/v1.0/applications/ {"source_id": "3", "application_type_id": "2"} Response: { "id": 3, "source_id": 3, "application_type_id": 2, "created_at": "2021-01-22T14:36:04.739Z", "updated_at": "2021-01-22T14:36:04.739Z", "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "extra": {}, "tenant": "10001" } PATCH http://localhost:3000/api/v1.0/applications/3/ {"extra": {"bucket": "cost-usage-bucket"}} Response: 204.. POST http://localhost:3000/api/v1.0/authentications/ { "authtype": "arn", "name": "", "password": "arn:aws:iam::999999999999:role/CostManagement", "status": "valid", "status_details": "Details Here", "username": "username", "resource_type": "Application", "resource_id": "3" } Response: { "id": 3, "resource_type": "Application", "resource_id": 3, "name": "", "authtype": "arn", "username": "username", "password": "arn:aws:iam::999999999999:role/CostManagement", "status": "valid", "status_details": "Details Here", "extra": null, "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "source_id": 3, "tenant": "10001" } Source successfully created GET /api/cost-management/v1/sources/3/ HTTP 200 OK Allow: GET, HEAD, PATCH, DELETE Cache-Control: max-age=0, no-cache, no-store, must-revalidate, private Content-Type: application/json Expires: Fri, 22 Jan 2021 14:38:26 GMT Vary: Accept { "id": 3, "uuid": "f227d4b1-6b8f-42bd-9d7c-cd51ac55dd86", "name": "AWS Source App-Extra", "source_type": "AWS", "authentication": { "credentials": { "role_arn": "arn:aws:iam::999999999999:role/CostManagement" } }, "billing_source": { "data_source": { "bucket": "cost-usage-bucket" } }, "provider_linked": true, "active": true, "current_month_data": true, "infrastructure": "Unknown", "cost_models": [] } sources_api_development=# select id, availability_status from applications where id=3; id | availability_status ----+--------------------- 3 | available (1 row) sources_api_development=# 4. Add Azure source with Application-extra flow POST http://localhost:3000/api/v1.0/sources/ {"source_type_id": "3", "name": "Azure Source App-Extra"} Response: { "id": 4, "name": "Azure Source App-Extra", "uid": "e489ae01-c610-4f7b-a66e-3ae7957a9752", "created_at": "2021-01-22T14:40:22.647Z", "updated_at": "2021-01-22T14:40:22.647Z", "source_type_id": 3, "version": null, "availability_status": null, "imported": null, "source_ref": null, "last_checked_at": null, "last_available_at": null, "tenant": "10001" } POST http://localhost:3000/api/v1.0/applications/ {"source_id": "4", "application_type_id": "2"} Response: { "id": 4, "source_id": 4, "application_type_id": 2, "created_at": "2021-01-22T14:41:00.803Z", "updated_at": "2021-01-22T14:41:00.803Z", "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "extra": {}, "tenant": "10001" } PATCH http://localhost:3000/api/v1.0/applications/4/ {"extra": {"subscription_id": "redacted-sub-id", "resource_group": "redacted-rg", "storage_account": "redacted-sa"}} Response: 204.. POST http://localhost:3000/api/v1.0/authentications/ { "authtype": "tenant_id_client_id_client_secret", "name": "", "password": "redacted-secret", "status": "valid", "status_details": "Details Here", "username": "redacted-client-id", "resource_type": "Application", "extra": {"azure": {"tenant_id": "redacted-tenant-id"}}, "resource_id": "4" } Response: { "id": 4, "resource_type": "Application", "resource_id": 4, "name": "", "authtype": "tenant_id_client_id_client_secret", "username": "redacted-client-id", "password": "redacted-secret", "status": "valid", "status_details": "Details Here", "extra": { "azure": { "tenant_id": "redacted-tenant-id" } }, "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "source_id": 4, "tenant": "10001" } Source successfully created: GET /api/cost-management/v1/sources/4/ HTTP 200 OK Allow: GET, HEAD, PATCH, DELETE Cache-Control: max-age=0, no-cache, no-store, must-revalidate, private Content-Type: application/json Expires: Fri, 22 Jan 2021 14:42:41 GMT Vary: Accept { "id": 4, "uuid": "e489ae01-c610-4f7b-a66e-3ae7957a9752", "name": "Azure Source App-Extra", "source_type": "Azure", "authentication": { "credentials": { "client_id": "redacted-client-id", "tenant_id": "redacted-tenant-id", "subscription_id": "redacted-sub-id" } }, "billing_source": { "data_source": { "resource_group": "redacted-rg", "storage_account": "redacted-sa" } }, "provider_linked": true, "active": true, "current_month_data": true, "infrastructure": "Unknown", "cost_models": [] } sources_api_development=# select id, availability_status from applications where id=4; id | availability_status ----+--------------------- 4 | available (1 row) sources_api_development=# 5. Update AWS bucket for source_id =3 (Application.extra path) to an incorrect bucket and verify status is updated accordingly PATCH http://localhost:3000/api/v1.0/applications/3/ {"extra": {"bucket": "cost-usage-bucket-wrong"}} sources_api_development=# select id, availability_status, availability_status_error from applications where id=3; id | availability_status | availability_status_error ----+---------------------+-------------------------------------------------------------------------------------------------------------------- 3 | unavailable | The S3 bucket was entered incorrectly for this source. Edit your AWS source and verify the name of your S3 bucket. (1 row) Now fix it... PATCH http://localhost:3000/api/v1.0/applications/3/ {"extra": {"bucket": "cost-usage-bucket"}} sources_api_development=# sources_api_development=# select id, availability_status, availability_status_error from applications where id=3; id | availability_status | availability_status_error ----+---------------------+--------------------------- 3 | available | (1 row) sources_api_development=# 5. Create AWS source with application-extra path and add bucket at the same time we create the application. Note: removing AWS source first.. POST http://localhost:3000/api/v1.0/sources/ {"source_type_id": "1", "name": "AWS Source App-Extra"} Response: { "id": 5, "name": "AWS Source App-Extra", "uid": "61b90af5-9e68-4390-9df7-1f037523d090", "created_at": "2021-01-22T14:46:21.788Z", "updated_at": "2021-01-22T14:46:21.788Z", "source_type_id": 1, "version": null, "availability_status": null, "imported": null, "source_ref": null, "last_checked_at": null, "last_available_at": null, "tenant": "10001" } POST http://localhost:3000/api/v1.0/applications/ {"source_id": "5", "application_type_id": "2", "extra": {"bucket": "cost-usage-bucket"}} Response: { "id": 5, "source_id": 5, "application_type_id": 2, "created_at": "2021-01-22T14:47:04.545Z", "updated_at": "2021-01-22T14:47:04.545Z", "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "extra": { "bucket": "cost-usage-bucket" }, "tenant": "10001" } POST http://localhost:3000/api/v1.0/authentications/ { "authtype": "arn", "name": "", "password": "arn:aws:iam::999999999999:role/CostManagement", "status": "valid", "status_details": "Details Here", "username": "username", "resource_type": "Application", "resource_id": "5" } Response: { "id": 5, "resource_type": "Application", "resource_id": 5, "name": "", "authtype": "arn", "username": "username", "password": "arn:aws:iam::999999999999:role/CostManagement", "status": "valid", "status_details": "Details Here", "extra": null, "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "source_id": 5, "tenant": "10001" } Source successfully created: GET /api/cost-management/v1/sources/5/ HTTP 200 OK Allow: GET, HEAD, PATCH, DELETE Cache-Control: max-age=0, no-cache, no-store, must-revalidate, private Content-Type: application/json Expires: Fri, 22 Jan 2021 14:48:33 GMT Vary: Accept { "id": 5, "uuid": "61b90af5-9e68-4390-9df7-1f037523d090", "name": "AWS Source App-Extra", "source_type": "AWS", "authentication": { "credentials": { "role_arn": "arn:aws:iam::999999999999:role/CostManagement" } }, "billing_source": { "data_source": { "bucket": "cost-usage-bucket" } }, "provider_linked": true, "active": true, "current_month_data": true, "infrastructure": "Unknown", "cost_models": [] } sources_api_development=# select id, availability_status, availability_status_error from applications where id=5; id | availability_status | availability_status_error ----+---------------------+--------------------------- 5 | available | (1 row) sources_api_development=# 6. Add GCP source (using WIP sources branch) with an invalid project name (verify status is correct). Fix project name and verify source is created successfully. POST http://localhost:3000/api/v1.0/sources/ {"source_type_id": "6", "name": "GCP Source"} Response: { "id": 6, "name": "GCP Source", "uid": "b3560529-fdba-43bb-927f-b74a98e043ec", "created_at": "2021-01-22T14:50:52.177Z", "updated_at": "2021-01-22T14:50:52.177Z", "source_type_id": 6, "version": null, "availability_status": null, "imported": null, "source_ref": null, "last_checked_at": null, "last_available_at": null, "tenant": "10001" } POST http://localhost:3000/api/v3.0/applications/ {"source_id": "6", "application_type_id": "2"} Response: { "id": 6, "source_id": 6, "application_type_id": 2, "created_at": "2021-01-22T14:51:15.980Z", "updated_at": "2021-01-22T14:51:15.980Z", "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "extra": {}, "tenant": "10001" } POST http://localhost:3000/api/v3.0/applications/6/ {"extra": {"dataset": "billing_datset"}} Response: 204.. POST http://localhost:3000/api/v1.0/authentications/ { "authtype": "project_id_service_account_json", "name": "", "password": "", "status": "valid", "status_details": "Details Here", "username": "wrong-project-id", "resource_type": "Application", "resource_id": "6" } { "id": 6, "resource_type": "Application", "resource_id": 6, "name": "", "authtype": "project_id_service_account_json", "username": "wrong-project-id", "password": "", "status": "valid", "status_details": "Details Here", "extra": null, "availability_status": null, "availability_status_error": null, "last_checked_at": null, "last_available_at": null, "source_id": 6, "tenant": "10001" } sources_api_development=# select id, availability_status, availability_status_error from applications where id=6; id | availability_status | availability_status_error ----+---------------------+------------------------------------------------------------- 6 | unavailable | Incorrect IAM permissions for project wrong-project-id (1 row) sources_api_development=# Fix project-id PATCH http://localhost:3000/api/v1.0/authentications/6/ {"username": "redacted-project-id"} Source Created Successfully: GET /api/cost-management/v1/sources/6/ HTTP 200 OK Allow: GET, HEAD, PATCH, DELETE Cache-Control: max-age=0, no-cache, no-store, must-revalidate, private Content-Type: application/json Expires: Fri, 22 Jan 2021 14:55:12 GMT Vary: Accept { "id": 6, "uuid": "b3560529-fdba-43bb-927f-b74a98e043ec", "name": "GCP Source", "source_type": "GCP", "authentication": { "credentials": { "project_id": "redacted-project-id" } }, "billing_source": { "data_source": { "dataset": "billing_datset", "table_id": "gcp_billing_export_v1_redacted" } }, "provider_linked": true, "active": true, "current_month_data": true, "infrastructure": "Unknown", "cost_models": [] } sources_api_development=# select id, availability_status, availability_status_error from applications where id=6; id | availability_status | availability_status_error ----+---------------------+--------------------------- 6 | available | (1 row) sources_api_development=#