From 945900830ef15c2c8e0fe5a4fbb686e3f1ae8a1e Mon Sep 17 00:00:00 2001 From: agelostsal Date: Tue, 13 Sep 2022 14:26:27 +0300 Subject: [PATCH] AM-284 API Call: Per project_admin usage report --- doc/swagger/swagger.yaml | 37 ++++++ handlers/metrics.go | 139 +++++++++++++++++++++++ handlers/metrics_test.go | 100 ++++++++++++++++ metrics/models.go | 5 + metrics/queries.go | 19 ++++ routing.go | 6 +- website/docs/api_advanced/api_metrics.md | 130 ++++++++++++++++++++- 7 files changed, 431 insertions(+), 5 deletions(-) diff --git a/doc/swagger/swagger.yaml b/doc/swagger/swagger.yaml index 25b9690b..6291588a 100644 --- a/doc/swagger/swagger.yaml +++ b/doc/swagger/swagger.yaml @@ -1250,6 +1250,32 @@ paths: 500: $ref: "#/responses/500" + /users/usageReport: + get: + summary: Show the projects and their metrics, alongside service operational metrics based on the provided key + description: | + Show a specific's user usage information regarding its projects + parameters: + - $ref: '#/parameters/ApiKey' + - $ref: '#/parameters/StartDate' + - $ref: '#/parameters/EndDate' + - $ref: '#/parameters/Projects' + tags: + - Users + responses: + 200: + description: A User usage report object + schema: + $ref: '#/definitions/UserUsageReport' + 400: + $ref: "#/responses/400" + 401: + $ref: "#/responses/401" + 403: + $ref: "#/responses/403" + 500: + $ref: "#/responses/500" + /users/{USER}:refreshToken: post: summary: Refreshes the token of an existing user @@ -2563,6 +2589,17 @@ definitions: schema: type: object + UserUsageReport: + type: object + properties: + va_report: + type: object + $ref: '#/definitions/VAMetrics' + operational_metrics: + type: array + items: + $ref: '#/definitions/Metric' + VAMetrics: type: object properties: diff --git a/handlers/metrics.go b/handlers/metrics.go index 197b0616..4e6c57fe 100644 --- a/handlers/metrics.go +++ b/handlers/metrics.go @@ -3,6 +3,7 @@ package handlers import ( "encoding/json" "fmt" + "github.com/ARGOeu/argo-messaging/config" "net/http" "strings" "time" @@ -122,6 +123,144 @@ func VaMetrics(w http.ResponseWriter, r *http.Request) { respondOK(w, output) } +// UserUsageReport (GET) retrieves metrics regarding a project's users, subscriptions, topics and messages +// alongside service operational metrics +// This handler is supposed to be used for project admins in order to get usage information for their projects +func UserUsageReport(w http.ResponseWriter, r *http.Request) { + + // Add content type header to the response + contentType := "application/json" + charset := "utf-8" + w.Header().Add("Content-Type", fmt.Sprintf("%s; charset=%s", contentType, charset)) + + // Grab context references + refStr := gorillaContext.Get(r, "str").(stores.Store) + + authOption := gorillaContext.Get(r, "authOption").(config.AuthOption) + + tokenExtractStrategy := GetRequestTokenExtractStrategy(authOption) + token := tokenExtractStrategy(r) + + if token == "" { + err := APIErrorUnauthorized() + respondErr(w, err) + return + } + + user, err := auth.GetUserByToken(token, refStr) + + if err != nil { + if err.Error() == "not found" { + err := APIErrorUnauthorized() + respondErr(w, err) + return + } + err := APIErrQueryDatastore() + respondErr(w, err) + return + } + + startDate := time.Time{} + endDate := time.Time{} + + // if no start date was provided, set it to the start of the unix time + if r.URL.Query().Get("start_date") != "" { + startDate, err = time.Parse("2006-01-02", r.URL.Query().Get("start_date")) + if err != nil { + err := APIErrorInvalidData("Start date is not in valid format") + respondErr(w, err) + return + } + } else { + startDate = time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC) + } + + // if no end date was provided, set it to today + if r.URL.Query().Get("end_date") != "" { + endDate, err = time.Parse("2006-01-02", r.URL.Query().Get("end_date")) + if err != nil { + err := APIErrorInvalidData("End date is not in valid format") + respondErr(w, err) + return + } + } else { + endDate = time.Now().UTC() + } + + if startDate.After(endDate) { + err := APIErrorInvalidData("Start date cannot be after the end date") + respondErr(w, err) + return + } + + // filter based on url parameters projects and project_admin role + projectsUrlValue := r.URL.Query().Get("projects") + projectsList := make([]string, 0) + if projectsUrlValue != "" { + projectsList = strings.Split(projectsUrlValue, ",") + } + + queryProjects := make([]string, 0) + + for _, p := range user.Projects { + // first check that the user is a project admin for the specific project + isProjectAdmin := false + for _, userRole := range p.Roles { + if userRole == "project_admin" { + isProjectAdmin = true + } + } + + if !isProjectAdmin { + continue + } + + // check if the project belongs to the filter list of projects + // first check if the filter has any value provided + if projectsUrlValue != "" { + for _, filterProject := range projectsList { + if filterProject == p.Project { + queryProjects = append(queryProjects, p.Project) + } + } + } else { + // add the project that the user is a project admin for + queryProjects = append(queryProjects, p.Project) + } + + } + + // if no projects were marked for the query return empty response + vr := metrics.UserUsageReport{ + VAReport: metrics.VAReport{ + ProjectsMetrics: metrics.TotalProjectsMessageCount{ + Projects: []metrics.ProjectMetrics{}, + }, + }, + OperationalMetrics: metrics.MetricList{ + Metrics: []metrics.Metric{}, + }, + } + if len(queryProjects) > 0 { + vr, err = metrics.GetUserUsageReport(queryProjects, startDate, endDate, refStr) + if err != nil { + err := APIErrorNotFound(err.Error()) + respondErr(w, err) + return + } + } + + output, err := json.MarshalIndent(vr, "", " ") + if err != nil { + err := APIErrExportJSON() + respondErr(w, err) + return + } + + // Write response + respondOK(w, output) +} + // ProjectMetrics (GET) metrics for one project (number of topics) func ProjectMetrics(w http.ResponseWriter, r *http.Request) { diff --git a/handlers/metrics_test.go b/handlers/metrics_test.go index 30a17608..8ed814c1 100644 --- a/handlers/metrics_test.go +++ b/handlers/metrics_test.go @@ -1,12 +1,14 @@ package handlers import ( + "encoding/json" "io/ioutil" "net/http" "net/http/httptest" "strconv" "strings" "testing" + "time" "github.com/ARGOeu/argo-messaging/brokers" "github.com/ARGOeu/argo-messaging/config" @@ -719,6 +721,104 @@ func (suite *MetricsHandlersTestSuite) TestTopicMetricsNotFound() { } +func (suite *MetricsHandlersTestSuite) TestUserUsageProfile() { + + req, err := http.NewRequest("GET", "http://localhost:8080/v1/users/usageReport"+ + "?projects=ARGO&start_date=2007-10-01&end_date=2020-11-24&key=S3CR3T1T", nil) + if err != nil { + log.Fatal(err) + } + + expResp := `{ + "va_metrics": { + "projects_metrics": { + "projects": [ + { + "project": "ARGO", + "message_count": 140, + "average_daily_messages": 0, + "topics_count": 4, + "subscriptions_count": 4, + "users_count": 7 + } + ], + "total_message_count": 140, + "average_daily_messages": 0 + }, + "total_users_count": 7, + "total_topics_count": 4, + "total_subscriptions_count": 4 + }, + "operational_metrics": { + "metrics": [ + { + "metric": "ams_node.cpu_usage", + "metric_type": "percentage", + "value_type": "float64", + "resource_type": "ams_node", + "resource_name": "{{HOST}}", + "timeseries": [ + { + "timestamp": "{{TS1}}", + "value": {{VAL1}} + } + ], + "description": "Percentage value that displays the CPU usage of ams service in the specific node" + }, + { + "metric": "ams_node.memory_usage", + "metric_type": "percentage", + "value_type": "float64", + "resource_type": "ams_node", + "resource_name": "{{HOST}}", + "timeseries": [ + { + "timestamp": "{{TS1}}", + "value": {{VAL2}} + } + ], + "description": "Percentage value that displays the Memory usage of ams service in the specific node" + } + ] + } +}` + + cfgKafka := config.NewAPICfg() + cfgKafka.LoadStrJSON(suite.cfgStr) + brk := brokers.MockBroker{} + str := stores.NewMockStore("whatever", "argo_mgs") + str.UserList = append(str.UserList, stores.QUser{1, "uuid1", []stores.QProjectRoles{ + { + ProjectUUID: "argo_uuid", + Roles: []string{"project_admin"}, + }, + }, + "UserA", "FirstA", "LastA", "OrgA", + "DescA", "S3CR3T1T", "foo-email", []string{}, + time.Now(), time.Now(), ""}) + router := mux.NewRouter().StrictSlash(true) + w := httptest.NewRecorder() + mgr := oldPush.Manager{} + router.HandleFunc("/v1/users/usageReport", WrapMockAuthConfig(UserUsageReport, cfgKafka, &brk, str, &mgr, nil)) + router.ServeHTTP(w, req) + + metricOut := metrics.UserUsageReport{} + json.Unmarshal([]byte(w.Body.String()), &metricOut) + ts1 := metricOut.OperationalMetrics.Metrics[0].Timeseries[0].Timestamp + val1 := metricOut.OperationalMetrics.Metrics[0].Timeseries[0].Value.(float64) + ts2 := metricOut.OperationalMetrics.Metrics[1].Timeseries[0].Timestamp + val2 := metricOut.OperationalMetrics.Metrics[1].Timeseries[0].Value.(float64) + host := metricOut.OperationalMetrics.Metrics[0].Resource + expResp = strings.Replace(expResp, "{{TS1}}", ts1, -1) + expResp = strings.Replace(expResp, "{{TS2}}", ts2, -1) + expResp = strings.Replace(expResp, "{{VAL1}}", strconv.FormatFloat(val1, 'g', 1, 64), -1) + expResp = strings.Replace(expResp, "{{VAL2}}", strconv.FormatFloat(val2, 'g', 1, 64), -1) + expResp = strings.Replace(expResp, "{{HOST}}", host, -1) + + suite.Equal(200, w.Code) + suite.Equal(expResp, w.Body.String()) +} + func TestMetricsHandlersTestSuite(t *testing.T) { log.SetOutput(ioutil.Discard) suite.Run(t, new(MetricsHandlersTestSuite)) diff --git a/metrics/models.go b/metrics/models.go index 35f2c741..3f13d004 100644 --- a/metrics/models.go +++ b/metrics/models.go @@ -84,6 +84,11 @@ type Timepoint struct { Value interface{} `json:"value"` } +type UserUsageReport struct { + VAReport VAReport `json:"va_metrics"` + OperationalMetrics MetricList `json:"operational_metrics"` +} + // ExportJSON exports whole ProjectTopic structure func (m *Metric) ExportJSON() (string, error) { output, err := json.MarshalIndent(m, "", " ") diff --git a/metrics/queries.go b/metrics/queries.go index 3df6f03b..855b4a61 100644 --- a/metrics/queries.go +++ b/metrics/queries.go @@ -225,3 +225,22 @@ func GenerateVAReport(projects []string, startDate time.Time, endDate time.Time, return vaReport, nil } + +// GetUserUsageReport returns a VAReport populated with the needed metrics alongside service operational metrics +func GetUserUsageReport(projects []string, startDate time.Time, endDate time.Time, str stores.Store) (UserUsageReport, error) { + + vr, err := GetVAReport(projects, startDate, endDate, str) + if err != nil { + return UserUsageReport{}, err + } + + om, err := GetUsageCpuMem(str) + if err != nil { + return UserUsageReport{}, err + } + + return UserUsageReport{ + VAReport: vr, + OperationalMetrics: om, + }, nil +} diff --git a/routing.go b/routing.go index b76a3ba0..75cfb793 100644 --- a/routing.go +++ b/routing.go @@ -50,7 +50,10 @@ func NewRouting(cfg *config.APICfg, brk brokers.Broker, str stores.Store, mgr *o handler = handlers.WrapLog(handler, route.Name) // skip authentication/authorization for the health status and profile api calls - if route.Name != "ams:healthStatus" && "users:profile" != route.Name && route.Name != "version:list" { + if route.Name != "ams:healthStatus" && + "users:profile" != route.Name && + route.Name != "version:list" && + route.Name != "users:usageReport" { handler = handlers.WrapAuthorize(handler, route.Name, tokenExtractStrategy) handler = handlers.WrapAuthenticate(handler, tokenExtractStrategy) } @@ -86,6 +89,7 @@ var defaultRoutes = []APIRoute{ {"users:byUUID", "GET", "/users:byUUID/{uuid}", handlers.UserListByUUID}, {"users:list", "GET", "/users", handlers.UserListAll}, {"users:profile", "GET", "/users/profile", handlers.UserProfile}, + {"users:usageReport", "GET", "/users/usageReport", handlers.UserUsageReport}, {"users:show", "GET", "/users/{user}", handlers.UserListOne}, {"users:refreshToken", "POST", "/users/{user}:refreshToken", handlers.RefreshToken}, {"users:create", "POST", "/users/{user}", handlers.UserCreate}, diff --git a/website/docs/api_advanced/api_metrics.md b/website/docs/api_advanced/api_metrics.md index 22750e20..80726893 100644 --- a/website/docs/api_advanced/api_metrics.md +++ b/website/docs/api_advanced/api_metrics.md @@ -102,7 +102,7 @@ Success Response ### Errors Please refer to section [Errors](/api_basic/api_errors.md) to see all possible Errors -## [GET] Get Daily Message Average +## [GET] Get a VA Report This request returns the total amount of messages per project for the given time window. The number of messages is calculated using the `daily message count` for each one of the project's topics. @@ -121,14 +121,14 @@ GET "/v1/metrics/daily-message-average" ```bash curl -H "Content-Type: application/json" - "https://{URL}/v1/metrics/daily-message-average" + "https://{URL}/v1/metrics/va_metrics" ``` ### Example request with URL parameters ```bash curl -H "Content-Type: application/json" - "https://{URL}/v1/metrics/daily-message-average?start_date=2019-03-01&end_date=2019-07-24&projects=ARGO,ARGO-2" + "https://{URL}/v1/metrics/va_metrics?start_date=2019-03-01&end_date=2019-07-24&projects=ARGO,ARGO-2" ``` ### Responses @@ -156,4 +156,126 @@ Success Response } ``` ### Errors -Please refer to section [Errors](/api_basic/api_errors.md) to see all possible Errors \ No newline at end of file +Please refer to section [Errors](/api_basic/api_errors.md) to see all possible Errors + +## [GET] Get User usage report + +This is a combination of the va_metrics and the operational_metrics +api calls.The user will receive data for all of the projects that has +the project_admin role alongisde the operational metrics of the service. + +### Request +``` +GET "/v1/users/usageReport" + +``` +### URL parameters +`start_date`: start date for querying projects topics daily message count(optional), default value is the start unix time +`end_date`: start date for querying projects topics daily message count(optional), default is the time of the api call +`projects`: which projects to include to the query(optional), default is all registered projects + +### Example request + +```bash +curl -H "Content-Type: application/json" + "https://{URL}/v1/users/usageReport" +``` + +### Example request with URL parameters + +```bash +curl -H "Content-Type: application/json" + "https://{URL}/v1/users/usageReport?start_date=2019-03-01&end_date=2019-07-24&projects=ARGO,ARGO-2" +``` + +### Responses +Success Response +`200 OK` + +```json +{ + "va_metrics": { + "projects_metrics": { + "projects": [ + { + "project": "e2epush", + "message_count": 27, + "average_daily_messages": 0.03, + "topics_count": 3, + "subscriptions_count": 6, + "users_count": 0 + } + ], + "total_message_count": 27, + "average_daily_messages": 0.03 + }, + "total_users_count": 0, + "total_topics_count": 3, + "total_subscriptions_count": 6 + }, + "operational_metrics": { + "metrics": [ + { + "metric": "ams_node.cpu_usage", + "metric_type": "percentage", + "value_type": "float64", + "resource_type": "ams_node", + "resource_name": "test-MBP", + "timeseries": [ + { + "timestamp": "2022-09-13T09:39:56Z", + "value": 0 + } + ], + "description": "Percentage value that displays the CPU usage of ams service in the specific node" + }, + { + "metric": "ams_node.memory_usage", + "metric_type": "percentage", + "value_type": "float64", + "resource_type": "ams_node", + "resource_name": "test-MBP", + "timeseries": [ + { + "timestamp": "2022-09-13T09:39:56Z", + "value": 0.1 + } + ], + "description": "Percentage value that displays the Memory usage of ams service in the specific node" + }, + { + "metric": "ams_node.cpu_usage", + "metric_type": "percentage", + "value_type": "float64", + "resource_type": "ams_node", + "resource_name": "4", + "timeseries": [ + { + "timestamp": "2022-09-13T09:39:56Z", + "value": 0 + } + ], + "description": "Percentage value that displays the CPU usage of ams service in the specific node" + }, + { + "metric": "ams_node.memory_usage", + "metric_type": "percentage", + "value_type": "float64", + "resource_type": "ams_node", + "resource_name": "4", + "timeseries": [ + { + "timestamp": "2022-09-13T09:39:56Z", + "value": 0.1 + } + ], + "description": "Percentage value that displays the Memory usage of ams service in the specific node" + } + ] + } +} +``` +### Errors +Please refer to section [Errors](/api_basic/api_errors.md) to see all possible Errors + +