From fde925d6eb1883c35ba5d7d23fe6859128efd053 Mon Sep 17 00:00:00 2001 From: Vivek Singh Chauhan Date: Thu, 8 Aug 2024 10:46:10 -0700 Subject: [PATCH] APIGOV-28475 - TA config optimization (#817) * APIGOV-28475 - TA config optimization - Deprecate USAGEREPORTING_INTERVAL - New config for setting schedule for metric event, default to an hour - Remove TRACEABILITY_SAMPLING_REPORTALLERRORS - Added TRACEABILITY_SAMPLING_ONLYERRORS, default to false, when set only errors are sampled - Updated sampling counting to include errors * APIGOV-28475 - updated documentation * APIGOV-28475 - usage reporting schedule config - Deprecate CENTRAL_USAGEREPORTING_USAGESCHEDULE - Added CENTRAL_USAGEREPORTING_SCHEDULE * APIGOV-28475 - update QA usage config * APIGOV-28475 - update * APIGOV-28475 - fix to use overrides * INT - update kin-openapi --------- Co-authored-by: Jason Collins --- docs/traceability/index.md | 27 +-- go.mod | 8 +- go.sum | 17 +- pkg/config/centralconfig.go | 83 +++++---- pkg/config/metricreportingconfig.go | 174 ++++++++++++++++++ pkg/config/metricreportingconfig_test.go | 148 +++++++++++++++ pkg/config/usagereportingconfig.go | 169 ++++++----------- pkg/config/usagereportingconfig_test.go | 70 ++++--- pkg/traceability/sampling/definitions.go | 2 +- pkg/traceability/sampling/globalsampling.go | 12 +- pkg/traceability/sampling/sample.go | 6 +- pkg/traceability/sampling/sampling_test.go | 154 ++++++++++------ pkg/traceability/traceability_test.go | 2 +- pkg/transaction/eventgenerator_test.go | 1 + pkg/transaction/logeventbuilder.go | 2 +- pkg/transaction/metric/cachestorage.go | 6 +- pkg/transaction/metric/metricscollector.go | 19 +- .../metric/metricscollector_test.go | 9 +- pkg/transaction/metric/reportcache.go | 4 +- pkg/transaction/metric/usagepublisher.go | 2 +- 20 files changed, 620 insertions(+), 295 deletions(-) create mode 100644 pkg/config/metricreportingconfig.go create mode 100644 pkg/config/metricreportingconfig_test.go diff --git a/docs/traceability/index.md b/docs/traceability/index.md index 1f22b5662..5669527ae 100644 --- a/docs/traceability/index.md +++ b/docs/traceability/index.md @@ -74,10 +74,9 @@ Below is the list of Central configuration properties in YAML and their correspo | central.ssl.minVersion | CENTRAL_SSL_MINVERSION | String value for the minimum SSL/TLS version that is acceptable. If zero, empty TLS 1.0 is taken as the minimum. Allowed values are: TLS1.0, TLS1.1, TLS1.2, TLS1.3. | | central.ssl.maxVersion | CENTRAL_SSL_MAXVERSION | String value for the maximum SSL/TLS version that is acceptable. If empty, then the maximum version supported by this package is used, which is currently TLS 1.3. Allowed values are: TLS1.0, TLS1.1, TLS1.2, TLS1.3. | | central.proxyURL | CENTRAL_PROXYURL | The URL for the proxy for Amplify Central``. If empty, no proxy is defined. | -| central.usageReporting.url | CENTRAL_USAGEREPORTING_URL | The Lighthouse URL the agent publishes usage reports. See [Traceability usage reporting](#traceability-usage-reporting) | +| central.metricReporting.publish | CENTRAL_METRICREPORTING_PUBLISH | Enables/disables the sending of metric events to Amplify (default: `true`) | +| central.metricReporting.schedule | CENTRAL_METRICREPORTING_SCHEDULE | The frequency schedule in which the agent sends metric events to Amplify Central (default: `@hourly`) | | central.usageReporting.publish | CENTRAL_USAGEREPORTING_PUBLISH | Enables/disables the sending of usage events to Amplify (default: `true`) | -| central.usageReporting.publishMetric | CENTRAL_USAGEREPORTING_PUBLISHMETRIC | Enables/disables the sending of metric events to Amplify (default: `true`) | -| central.usageReporting.interval | CENTRAL_USAGEREPORTING_INTERVAL | The frequency in which the agent published activity to Amplify Central (default: `15m`) | | central.usageReporting.offline | CENTRAL_USAGEREPORTING_OFFLINE | Enables/disables the sending of usage events to lighthouse or saving to disk (default: `false`) | | central.usageReporting.offlineschedule | CENTRAL_USAGEREPORTING_OFFLINESCHEDULE | The frequency schedule that the agent collects activity for the offline reports (default: `@hourly`) | | central.grpc.enabled | CENTRAL_GRPC_ENABLED | Controls whether an agent uses a gRPC based stream connection to manage its internal cache. (Default value = false) | @@ -979,27 +978,31 @@ By default all transaction data is sent to Amplify. Below is the list of the sampling configuration properties in a YAML and their corresponding environment variables that can be set to override the config in YAML. All of these are children of output.traceability.sampling -| YAML property | Variable name | Description | -|-----------------|---------------------------------------|------------------------------------------------------------------------------------------------------------------------------| -| percentage | TRACEABILITY_SAMPLING_PERCENTAGE | Defines the percentage of events (0-10. Decimals allowed. Any value above will default it to 1) that are sent to Amplify | -| per_api | TRACEABILITY_SAMPLING_PER_API | Defines if the percentage above is applied to all events or separate based on API ID in the event | -| reportAllErrors | TRACEABILITY_SAMPLING_REPORTALLERRORS | Defines if all error transaction events are sent to Amplify | +| YAML property | Variable name | Description | +|-----------------|---------------------------------------|-------------------------------------------------------------------------------------------------------------------------------| +| percentage | TRACEABILITY_SAMPLING_PERCENTAGE | Defines the percentage of events (0-10. Decimals allowed. Any value above will default it to 1) that are sent to Amplify | +| per_api | TRACEABILITY_SAMPLING_PER_API | Defines if the percentage above is applied to all events or separate based on API ID in the event | +| onlyErrors | TRACEABILITY_SAMPLING_ONLYERRORS | Defines if only error transaction events are sent to Amplify | ### Traceability usage reporting The Amplify Agents SDK has the ability to track API usages and report them back to the Amplify platform. -By default usage reporting is on but can be configured using the following information. +By default metric and usage reporting is on but can be configured using the following information. + +Below is the list of the metric reporting configuration properties, all of these properties are children of [[agent type]].central.metricReporting in the yaml. + +| YAML property | Variable name | Default | Description | +|-----------------|----------------------------------------|------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| publish | CENTRAL_METRICREPORTING_PUBLISH | `true` | Defines if individual API Metrics will be published to Amplify | +| schedule | CENTRAL_METRICREPORTING_SCHEDULE | `@hourly` | Defines the schedule, in the default online mode, that metric events are sent to Amplify | Below is the list of the usage reporting configuration properties, all of these properties are children of [[agent type]].central.usageReporting in the yaml. | YAML property | Variable name | Default | Description | |-----------------|----------------------------------------|------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| -| url | CENTRAL_USAGEREPORTING_URL | https://lighthouse.admin.axway.com | Defines the url on the Amplify platform that usage events are sent to | | publish | CENTRAL_USAGEREPORTING_PUBLISH | `true` | Defines if API usage numbers will be published to Amplify | -| publishMetric | CENTRAL_USAGEREPORTING_PUBLISHMETRIC | `true` | Defines if individual API Metrics will be published to Amplify | -| interval | CENTRAL_USAGEREPORTING_INTERVAL | _15m_ | Defines the interval, in the default online mode, that usage data is sent to Amplify | | offline | CENTRAL_USAGEREPORTING_OFFLINE | `false` | Defines if the agent is working in offline mode for generating usage reports, see [Offline usage reporting](#offline-usage-reporting) | | offlineSchedule | CENTRAL_USAGEREPORTING_OFFLINESCHEDULE | `@hourly` | Defines the schedule in which the usage numbers are determined when in offline mode, see [Defining a schedule](../../pkg/jobs/README.md#defining-a-schedule) | diff --git a/go.mod b/go.mod index c1dd37bc8..a8ab40a6c 100644 --- a/go.mod +++ b/go.mod @@ -9,12 +9,12 @@ require ( github.com/emicklei/proto v1.9.2 github.com/fsnotify/fsnotify v1.5.4 github.com/gabriel-vasile/mimetype v1.4.0 - github.com/getkin/kin-openapi v0.125.0 + github.com/getkin/kin-openapi v0.127.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/uuid v1.6.0 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/invopop/yaml v0.2.0 + github.com/invopop/yaml v0.3.1 github.com/lestrrat-go/jwx/v2 v2.0.21 github.com/opentracing/opentracing-go v1.2.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 @@ -50,8 +50,8 @@ require ( github.com/elastic/go-ucfg v0.8.8 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/fatih/color v1.16.0 // indirect - github.com/go-openapi/jsonpointer v0.20.2 // indirect - github.com/go-openapi/swag v0.22.8 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/go.sum b/go.sum index f7f69a1bf..61444af47 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= -github.com/getkin/kin-openapi v0.125.0 h1:jyQCyf2qXS1qvs2U00xQzkGCqYPhEhZDmSmVt65fXno= -github.com/getkin/kin-openapi v0.125.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= +github.com/getkin/kin-openapi v0.127.0 h1:Mghqi3Dhryf3F8vR370nN67pAERW+3a95vomb3MAREY= +github.com/getkin/kin-openapi v0.127.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -119,12 +119,12 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= -github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw= -github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= @@ -227,8 +227,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= -github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= +github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= github.com/jcchavezs/porto v0.1.0/go.mod h1:fESH0gzDHiutHRdX2hv27ojnOVFco37hg1W6E9EZF4A= github.com/jcchavezs/porto v0.6.0 h1:AgQLGwsXaxDkPj4Y+paFkVGLAR4n/1RRF0xV5UKinwg= github.com/jcchavezs/porto v0.6.0/go.mod h1:fESH0gzDHiutHRdX2hv27ojnOVFco37hg1W6E9EZF4A= @@ -778,7 +778,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/config/centralconfig.go b/pkg/config/centralconfig.go index d9edf77b8..9d23b206d 100644 --- a/pkg/config/centralconfig.go +++ b/pkg/config/centralconfig.go @@ -203,6 +203,7 @@ type CentralConfig interface { GetCatalogItemByIDURL(catalogItemID string) string GetAppendEnvironmentToTitle() bool GetUsageReportingConfig() UsageReportingConfig + GetMetricReportingConfig() MetricReportingConfig IsUsingGRPC() bool GetGRPCHost() string GetGRPCPort() int @@ -224,35 +225,36 @@ type CentralConfiguration struct { IConfigValidator AgentType AgentType RegionSettings regionalSettings - Region Region `config:"region"` - TenantID string `config:"organizationID"` - TeamName string `config:"team"` - APICDeployment string `config:"deployment"` - Environment string `config:"environment"` - EnvironmentID string `config:"environmentID"` - AgentName string `config:"agentName"` - URL string `config:"url"` - SingleURL string `config:"platformSingleURL"` - PlatformURL string `config:"platformURL"` - APIServerVersion string `config:"apiServerVersion"` - TagsToPublish string `config:"additionalTags"` - AppendEnvironmentToTitle bool `config:"appendEnvironmentToTitle"` - MigrationSettings MigrationConfig `config:"migration"` - Auth AuthConfig `config:"auth"` - TLS TLSConfig `config:"ssl"` - PollInterval time.Duration `config:"pollInterval"` - ReportActivityFrequency time.Duration `config:"reportActivityFrequency"` - ClientTimeout time.Duration `config:"clientTimeout"` - PageSize int `config:"pageSize"` - APIValidationCronSchedule string `config:"apiValidationCronSchedule"` - APIServiceRevisionPattern string `config:"apiServiceRevisionPattern"` - ProxyURL string `config:"proxyUrl"` - SubscriptionConfiguration SubscriptionConfig `config:"subscriptions"` - UsageReporting UsageReportingConfig `config:"usageReporting"` - GRPCCfg GRPCConfig `config:"grpc"` - CacheStoragePath string `config:"cacheStoragePath"` - CacheStorageInterval time.Duration `config:"cacheStorageInterval"` - CredentialConfig CredentialConfig `config:"credential"` + Region Region `config:"region"` + TenantID string `config:"organizationID"` + TeamName string `config:"team"` + APICDeployment string `config:"deployment"` + Environment string `config:"environment"` + EnvironmentID string `config:"environmentID"` + AgentName string `config:"agentName"` + URL string `config:"url"` + SingleURL string `config:"platformSingleURL"` + PlatformURL string `config:"platformURL"` + APIServerVersion string `config:"apiServerVersion"` + TagsToPublish string `config:"additionalTags"` + AppendEnvironmentToTitle bool `config:"appendEnvironmentToTitle"` + MigrationSettings MigrationConfig `config:"migration"` + Auth AuthConfig `config:"auth"` + TLS TLSConfig `config:"ssl"` + PollInterval time.Duration `config:"pollInterval"` + ReportActivityFrequency time.Duration `config:"reportActivityFrequency"` + ClientTimeout time.Duration `config:"clientTimeout"` + PageSize int `config:"pageSize"` + APIValidationCronSchedule string `config:"apiValidationCronSchedule"` + APIServiceRevisionPattern string `config:"apiServiceRevisionPattern"` + ProxyURL string `config:"proxyUrl"` + SubscriptionConfiguration SubscriptionConfig `config:"subscriptions"` + UsageReporting UsageReportingConfig `config:"usageReporting"` + MetricReporting MetricReportingConfig `config:"metricReporting"` + GRPCCfg GRPCConfig `config:"grpc"` + CacheStoragePath string `config:"cacheStoragePath"` + CacheStorageInterval time.Duration `config:"cacheStorageInterval"` + CredentialConfig CredentialConfig `config:"credential"` JobExecutionTimeout time.Duration environmentID string teamID string @@ -291,6 +293,7 @@ func NewCentralConfig(agentType AgentType) CentralConfig { ReportActivityFrequency: 5 * time.Minute, APIValidationCronSchedule: "@daily", UsageReporting: NewUsageReporting(platformURL), + MetricReporting: NewMetricReporting(), JobExecutionTimeout: 5 * time.Minute, CacheStorageInterval: 10 * time.Second, GRPCCfg: GRPCConfig{ @@ -630,6 +633,16 @@ func (c *CentralConfiguration) GetUsageReportingConfig() UsageReportingConfig { return c.UsageReporting } +// GetMetricReportingConfig - +func (c *CentralConfiguration) GetMetricReportingConfig() MetricReportingConfig { + // Some paths in DA are checking usage reporting . So return an empty usage reporting config if nil + // Find All References to see DA scenarios checking for this config + if c.MetricReporting == nil { + return NewMetricReporting() + } + return c.MetricReporting +} + // GetCredentialConfig - func (c *CentralConfiguration) GetCredentialConfig() CredentialConfig { return c.CredentialConfig @@ -778,6 +791,7 @@ func (c *CentralConfiguration) ValidateCfg() (err error) { } if supportsTraceability(c.AgentType) { + c.GetMetricReportingConfig().Validate() c.GetUsageReportingConfig().Validate() } }, @@ -927,6 +941,7 @@ func AddCentralConfigProperties(props properties.Properties, agentType AgentType if supportsTraceability(agentType) { props.AddStringProperty(pathEnvironmentID, "", "Offline Usage Reporting Only. The Environment ID the usage is associated with on Amplify Central") props.AddStringProperty(pathDeployment, "", "Amplify Central") + AddMetricReportingProperties(props) AddUsageReportingProperties(props) } else { props.AddStringProperty(pathAdditionalTags, "", "Additional Tags to Add to discovered APIs when publishing to Amplify Central") @@ -950,15 +965,18 @@ func ParseCentralConfig(props properties.Properties, agentType AgentType) (Centr // check if CENTRAL_SINGLEURL is explicitly empty _, set := os.LookupEnv("CENTRAL_SINGLEURL") + var metricReporting MetricReportingConfig var usageReporting UsageReportingConfig if supportsTraceability(agentType) { + metricReporting = ParseMetricReportingConfig(props) usageReporting = ParseUsageReportingConfig(props) if usageReporting.IsOfflineMode() { // Check if this is offline usage reporting only cfg := &CentralConfiguration{ - AgentName: props.StringPropertyValue(pathAgentName), - AgentType: agentType, - UsageReporting: usageReporting, + AgentName: props.StringPropertyValue(pathAgentName), + AgentType: agentType, + UsageReporting: usageReporting, + MetricReporting: metricReporting, } // only need the environment ID in offline mode cfg.EnvironmentID = props.StringPropertyValue(pathEnvironmentID) @@ -1021,6 +1039,7 @@ func ParseCentralConfig(props properties.Properties, agentType AgentType) (Centr if supportsTraceability(agentType) { cfg.APICDeployment = props.StringPropertyValue(pathDeployment) cfg.UsageReporting = usageReporting + cfg.MetricReporting = metricReporting } else { cfg.TeamName = props.StringPropertyValue(pathTeam) cfg.TagsToPublish = props.StringPropertyValue(pathAdditionalTags) diff --git a/pkg/config/metricreportingconfig.go b/pkg/config/metricreportingconfig.go new file mode 100644 index 000000000..ef2a84f04 --- /dev/null +++ b/pkg/config/metricreportingconfig.go @@ -0,0 +1,174 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "time" + + "github.com/Axway/agent-sdk/pkg/cmd/properties" + "github.com/Axway/agent-sdk/pkg/util/exception" + "github.com/Axway/agent-sdk/pkg/util/log" + "github.com/gorhill/cronexpr" +) + +const ( + // DEPRECATE remove old and new env vars as well as checks below + oldUsageReportingPublishMetricEnvVar = "CENTRAL_USAGEREPORTING_PUBLISHMETRIC" + oldUsageReportingIntervalEnvVar = "CENTRAL_USAGEREPORTING_INTERVAL" + newMetricReportingPublishEnvVar = "CENTRAL_METRICREPORTING_PUBLISH" + newMetricReportingScheduleEnvVar = "CENTRAL_METRICREPORTING_SCHEDULE" + + // Config paths + pathMetricReportingPublish = "central.metricreporting.publish" + pathMetricReportingSchedule = "central.metricreporting.schedule" + + qaMetricReportingScheduleEnvVar = "QA_CENTRAL_METRICREPORTING_SCHEDULE" +) + +// MetricReportingConfig - Interface to get metric reporting config +type MetricReportingConfig interface { + CanPublish() bool + GetSchedule() string + GetReportGranularity() int + UsingQAVars() bool + Validate() +} + +// MetricReportingConfiguration - structure to hold all metric reporting settings +type MetricReportingConfiguration struct { + MetricReportingConfig + Publish bool `config:"publish"` + Schedule string `config:"schedule"` + granularity time.Duration + qaVars bool +} + +// NewMetricReporting - Creates the default metric reporting config +func NewMetricReporting() MetricReportingConfig { + return &MetricReportingConfiguration{ + Publish: true, + Schedule: "@hourly", + granularity: time.Hour, + qaVars: false, + } +} + +// Validate - +func (m *MetricReportingConfiguration) Validate() { + m.validatePublish() + // Parse and validate interval from deprecated config for backward compatibility + m.validateInterval() + m.validateSchedule() +} + +func (u *MetricReportingConfiguration) validateInterval() { + if val := os.Getenv(newMetricReportingScheduleEnvVar); val != "" { + return // this env var is set use what has been parsed + } + + // check if the old env var had a value + if val := os.Getenv(oldUsageReportingIntervalEnvVar); val != "" { + if value, err := time.ParseDuration(val); err == nil { + log.DeprecationWarningReplace(oldUsageReportingIntervalEnvVar, newMetricReportingScheduleEnvVar) + if value < 60*time.Second { + exception.Throw(ErrBadConfig.FormatError(oldUsageReportingIntervalEnvVar)) + } + + intervalSchedule := fmt.Sprintf("*/%d * * * *", int(value.Minutes())) + if value > time.Hour { + intervalSchedule = fmt.Sprintf("%d %d * * *", time.Now().Minute(), int(value.Hours())) + } + + _, err := cronexpr.Parse(intervalSchedule) + if err != nil { + exception.Throw(ErrBadConfig.FormatError(oldUsageReportingIntervalEnvVar)) + } + u.Schedule = intervalSchedule + } + } +} + +func (m *MetricReportingConfiguration) validatePublish() { + if val := os.Getenv(newMetricReportingPublishEnvVar); val != "" { + return // this env var is set use what has been parsed + } + + // check if the old env var had a value + if val := os.Getenv(oldUsageReportingPublishMetricEnvVar); val != "" { + if value, err := strconv.ParseBool(val); err == nil { + log.DeprecationWarningReplace(oldUsageReportingPublishMetricEnvVar, newMetricReportingPublishEnvVar) + m.Publish = value + } + } +} + +func (u *MetricReportingConfiguration) validateSchedule() { + // check if the qa env var is set + if val := os.Getenv(qaMetricReportingScheduleEnvVar); val != "" { + if _, err := cronexpr.Parse(val); err != nil { + log.Tracef("Could not use %s (%s) it is not a proper cron schedule", qaMetricReportingScheduleEnvVar, val) + } else { + log.Tracef("Using %s (%s) rather than the default (%s) for non-QA", qaMetricReportingScheduleEnvVar, val, u.Schedule) + u.Schedule = val + u.qaVars = true + } + return + } + + // Check the cron expressions + cron, err := cronexpr.Parse(u.Schedule) + if err != nil { + exception.Throw(ErrBadConfig.FormatError(pathMetricReportingSchedule)) + } + checks := 5 + nextRuns := cron.NextN(time.Now(), uint(checks)) + if len(nextRuns) != checks { + exception.Throw(ErrBadConfig.FormatError(pathMetricReportingSchedule)) + } + for i := 1; i < checks-1; i++ { + u.granularity = nextRuns[i].Sub(nextRuns[i-1]) + if u.granularity < time.Minute*15 { + log.Tracef("%s must be at 15 min apart", pathMetricReportingSchedule) + exception.Throw(ErrBadConfig.FormatError(pathMetricReportingSchedule)) + } + } +} + +// CanPublish - Returns the publish metric boolean +func (u *MetricReportingConfiguration) CanPublish() bool { + return u.Publish +} + +// GetSchedule - Returns the schedule string +func (u *MetricReportingConfiguration) GetSchedule() string { + return u.Schedule +} + +// GetReportGranularity - Returns the schedule string +func (u *MetricReportingConfiguration) GetReportGranularity() int { + return int(u.granularity.Milliseconds()) +} + +// UsingQAVars - Returns the offline boolean +func (u *MetricReportingConfiguration) UsingQAVars() bool { + return u.qaVars +} + +// AddMetricReportingProperties - Adds the command properties needed for Metric Reporting Settings +func AddMetricReportingProperties(props properties.Properties) { + props.AddBoolProperty(pathMetricReportingPublish, true, "Indicates if the agent can publish metric events to Amplify platform. Default to true") + props.AddStringProperty(pathMetricReportingSchedule, "@hourly", "The schedule at metric events are sent to the platform") +} + +// ParseUsageReportingConfig - Parses the Usage Reporting Config values from the command line +func ParseMetricReportingConfig(props properties.Properties) MetricReportingConfig { + // Start with the default config + cfg := NewMetricReporting().(*MetricReportingConfiguration) + + // update the config + cfg.Publish = props.BoolPropertyValue(pathMetricReportingPublish) + cfg.Schedule = props.StringPropertyValue(pathMetricReportingSchedule) + + return cfg +} diff --git a/pkg/config/metricreportingconfig_test.go b/pkg/config/metricreportingconfig_test.go new file mode 100644 index 000000000..bf403abb2 --- /dev/null +++ b/pkg/config/metricreportingconfig_test.go @@ -0,0 +1,148 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "testing" + "time" + + "github.com/Axway/agent-sdk/pkg/cmd/properties" + "github.com/Axway/agent-sdk/pkg/util/exception" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +func validateMetricReporting(cfg MetricReportingConfig) (err error) { + exception.Block{ + Try: func() { + cfg.Validate() + }, + Catch: func(e error) { + err = e + }, + }.Do() + return +} + +type expectedMetricConfig struct { + publish bool + schedule string + granularity int + qaVars bool +} + +var defaultMetricConfigExpected = expectedMetricConfig{ + publish: true, + schedule: "@hourly", + granularity: int(time.Hour) * 1, + qaVars: false, +} + +func validateMetricConfig(t *testing.T, expVals expectedMetricConfig, cfg MetricReportingConfig) { + assert.Equal(t, expVals.publish, cfg.CanPublish()) + assert.Equal(t, expVals.schedule, cfg.GetSchedule()) + assert.Equal(t, expVals.qaVars, cfg.UsingQAVars()) +} + +func TestMetricReportingConfigEnvVarMigration(t *testing.T) { + rootCmd := &cobra.Command{ + Use: "test", + } + + props := properties.NewProperties(rootCmd) + AddMetricReportingProperties(props) + + // Test Interval old env vars + os.Setenv(oldUsageReportingIntervalEnvVar, "30m") + expected := defaultMetricConfigExpected + expected.schedule = "*/30 * * * *" + expected.granularity = int((30 * time.Minute).Milliseconds()) + + cfg := ParseMetricReportingConfig(props) + assert.NotNil(t, cfg) + err := validateMetricReporting(cfg) + assert.Nil(t, err) + validateMetricConfig(t, expected, cfg) + + // Test Interval in hour + os.Setenv(oldUsageReportingIntervalEnvVar, "2h") + expected.schedule = fmt.Sprintf("%d 2 * * *", time.Now().Minute()) + expected.granularity = int((2 * time.Hour).Milliseconds()) + + cfg = ParseMetricReportingConfig(props) + assert.NotNil(t, cfg) + err = validateMetricReporting(cfg) + + assert.Nil(t, err) + validateMetricConfig(t, expected, cfg) + + // Test Interval new env vars + os.Setenv(newMetricReportingScheduleEnvVar, defaultExpected.schedule) + + expected = defaultMetricConfigExpected + cfg = ParseMetricReportingConfig(props) + assert.NotNil(t, cfg) + err = validateMetricReporting(cfg) + assert.Nil(t, err) + validateMetricConfig(t, expected, cfg) + + // Test Publish old env vars + os.Setenv(oldUsageReportingPublishMetricEnvVar, "false") + expected = defaultMetricConfigExpected + expected.publish = false + + cfg = ParseMetricReportingConfig(props) + assert.NotNil(t, cfg) + err = validateMetricReporting(cfg) + assert.Nil(t, err) + validateMetricConfig(t, expected, cfg) + + // Test Publish new env vars + os.Setenv(newMetricReportingPublishEnvVar, strconv.FormatBool(defaultExpected.publish)) + + expected = defaultMetricConfigExpected + cfg = ParseMetricReportingConfig(props) + assert.NotNil(t, cfg) + err = validateMetricReporting(cfg) + assert.Nil(t, err) + validateMetricConfig(t, expected, cfg) +} + +func TestMetricReportingConfigProperties(t *testing.T) { + rootCmd := &cobra.Command{ + Use: "test", + } + + props := properties.NewProperties(rootCmd) + + // Test default config + AddMetricReportingProperties(props) + + cfg := ParseMetricReportingConfig(props) + assert.NotNil(t, cfg) + + err := validateMetricReporting(cfg) + assert.Nil(t, err) + + validateMetricConfig(t, defaultMetricConfigExpected, cfg) + + // invalid schedule + currentSchedule := cfg.GetSchedule() + cfg.(*MetricReportingConfiguration).Schedule = "*/1511 * * * *" + err = validateMetricReporting(cfg) + assert.NotNil(t, err) + cfg.(*MetricReportingConfiguration).Schedule = currentSchedule + + // QA schedule override + os.Setenv(qaMetricReportingScheduleEnvVar, "*/1 * * * *") + cfg.(*MetricReportingConfiguration).Schedule = "" + err = validateMetricReporting(cfg) + assert.Nil(t, err) +} + +func TestNewMetricReporting(t *testing.T) { + cfg := NewMetricReporting() + assert.NotNil(t, cfg) + validateMetricConfig(t, defaultMetricConfigExpected, cfg) +} diff --git a/pkg/config/usagereportingconfig.go b/pkg/config/usagereportingconfig.go index e3d2e2eee..938f3df5b 100644 --- a/pkg/config/usagereportingconfig.go +++ b/pkg/config/usagereportingconfig.go @@ -14,37 +14,31 @@ import ( const ( // DEPRECATE remove old and new env vars as well as checks below - oldUsageReportingPublishEnvVar = "CENTRAL_PUBLISHUSAGE" - oldUsageReportingPublishMetricEnvVar = "CENTRAL_PUBLISHMETRIC" - oldUsageReportingIntervalEnvVar = "CENTRAL_EVENTAGGREGATIONINTERVAL" - newUsageReportingPublishEnvVar = "CENTRAL_USAGEREPORTING_PUBLISH" - newUsageReportingPublishMetricEnvVar = "CENTRAL_USAGEREPORTING_PUBLISHMETRIC" - newUsageReportingIntervalEnvVar = "CENTRAL_USAGEREPORTING_INTERVAL" + oldUsageReportingPublishEnvVar = "CENTRAL_PUBLISHUSAGE" + newUsageReportingPublishEnvVar = "CENTRAL_USAGEREPORTING_PUBLISH" + oldUsageReportingScheduleEnvVar = "CENTRAL_USAGEREPORTING_USAGESCHEDULE" + newUsageReportingScheduleEnvVar = "CENTRAL_USAGEREPORTING_SCHEDULE" // QA EnvVars qaUsageReportingScheduleEnvVar = "QA_CENTRAL_USAGEREPORTING_OFFLINESCHEDULE" qaUsageReportingOfflineScheduleEnvVar = "QA_CENTRAL_USAGEREPORTING_OFFLINEREPORTSCHEDULE" - qaUsageReportingUsageScheduleEnvVar = "QA_CENTRAL_USAGEREPORTING_USAGESCHEDULE" + qaUsageReportingUsageScheduleEnvVar = "QA_CENTRAL_USAGEREPORTING_SCHEDULE" // Config paths - pathUsageReportingPublish = "central.usagereporting.publish" - pathUsageReportingPublishMetric = "central.usagereporting.publishMetric" - pathUsageReportingUsageSchedule = "central.usagereporting.usageSchedule" - pathUsageReportingInterval = "central.usagereporting.interval" - pathUsageReportingOffline = "central.usagereporting.offline" - pathUsageReportingSchedule = "central.usagereporting.offlineSchedule" + pathUsageReportingPublish = "central.usagereporting.publish" + pathUsageReportingSchedule = "central.usagereporting.schedule" + pathUsageReportingOffline = "central.usagereporting.offline" + pathUsageReportingOfflineSchedule = "central.usagereporting.offlineSchedule" ) // UsageReportingConfig - Interface to get usage reporting config type UsageReportingConfig interface { GetURL() string - CanPublishUsage() bool - CanPublishMetric() bool - GetInterval() time.Duration + CanPublish() bool GetReportInterval() time.Duration - GetUsageSchedule() string - IsOfflineMode() bool GetSchedule() string + IsOfflineMode() bool + GetOfflineSchedule() string GetReportSchedule() string GetReportGranularity() int UsingQAVars() bool @@ -54,12 +48,10 @@ type UsageReportingConfig interface { // UsageReportingConfiguration - structure to hold all usage reporting settings type UsageReportingConfiguration struct { UsageReportingConfig - Publish bool `config:"publish"` - PublishMetric bool `config:"publishMetric"` - Interval time.Duration `config:"interval"` - UsageSchedule string `config:"usageSchedule"` - Offline bool `config:"offline"` - Schedule string `config:"offlineSchedule"` + Publish bool `config:"publish"` + Schedule string `config:"schedule"` + Offline bool `config:"offline"` + OfflineSchedule string `config:"offlineSchedule"` URL string reportSchedule string reportGranularity int @@ -69,29 +61,13 @@ type UsageReportingConfiguration struct { // NewUsageReporting - Creates the default usage reporting config func NewUsageReporting(platformURL string) UsageReportingConfig { return &UsageReportingConfiguration{ - URL: platformURL, - Publish: true, - PublishMetric: true, - Interval: 15 * time.Minute, - UsageSchedule: "@daily", - Offline: false, - Schedule: "@hourly", - reportSchedule: "@monthly", - qaVars: false, - } -} - -func (u *UsageReportingConfiguration) validateInterval() { - if val := os.Getenv(newUsageReportingIntervalEnvVar); val != "" { - return // this env var is set use what has been parsed - } - - // check if the old env var had a value - if val := os.Getenv(oldUsageReportingIntervalEnvVar); val != "" { - if value, err := time.ParseDuration(val); err == nil { - log.DeprecationWarningReplace(oldUsageReportingIntervalEnvVar, newUsageReportingIntervalEnvVar) - u.Interval = value - } + URL: platformURL, + Publish: true, + Schedule: "@daily", + Offline: false, + OfflineSchedule: "@hourly", + reportSchedule: "@monthly", + qaVars: false, } } @@ -110,30 +86,9 @@ func (u *UsageReportingConfiguration) validatePublish() { } } -func (u *UsageReportingConfiguration) validatePublishMetric() { - if val := os.Getenv(newUsageReportingPublishMetricEnvVar); val != "" { - return // this env var is set use what has been parsed - } - - // check if the old env var had a value - if val := os.Getenv(oldUsageReportingPublishMetricEnvVar); val != "" { - if value, err := strconv.ParseBool(val); err == nil { - log.DeprecationWarningReplace(oldUsageReportingPublishMetricEnvVar, newUsageReportingPublishMetricEnvVar) - u.PublishMetric = value - } - } -} - // Validate - func (u *UsageReportingConfiguration) Validate() { - u.validateInterval() // DEPRECATE - eventAgg := u.Interval - if eventAgg < 60*time.Second { - exception.Throw(ErrBadConfig.FormatError(pathUsageReportingInterval)) - } - - u.validatePublish() // DEPRECATE - u.validatePublishMetric() + u.validatePublish() if !u.Offline { u.validateUsageSchedule() @@ -148,35 +103,42 @@ func (u *UsageReportingConfiguration) validateUsageSchedule() { if _, err := cronexpr.Parse(val); err != nil { log.Tracef("Could not use %s (%s) it is not a proper cron schedule", qaUsageReportingUsageScheduleEnvVar, val) } else { - log.Tracef("Using %s (%s) rather than the default (%s) for non-QA", qaUsageReportingUsageScheduleEnvVar, val, u.UsageSchedule) - u.UsageSchedule = val + log.Tracef("Using %s (%s) rather than the default (%s) for non-QA", qaUsageReportingUsageScheduleEnvVar, val, u.Schedule) + u.Schedule = val u.qaVars = true } return } + if val := os.Getenv(newUsageReportingScheduleEnvVar); val == "" { + if val = os.Getenv(oldUsageReportingScheduleEnvVar); val != "" { + log.DeprecationWarningReplace(oldUsageReportingScheduleEnvVar, newUsageReportingScheduleEnvVar) + u.Schedule = val + } + } + // Check the cron expressions - cron, err := cronexpr.Parse(u.UsageSchedule) + cron, err := cronexpr.Parse(u.Schedule) if err != nil { - exception.Throw(ErrBadConfig.FormatError(pathUsageReportingUsageSchedule)) + exception.Throw(ErrBadConfig.FormatError(pathUsageReportingSchedule)) } checks := 5 nextRuns := cron.NextN(time.Now(), uint(checks)) if len(nextRuns) != checks { - exception.Throw(ErrBadConfig.FormatError(pathUsageReportingUsageSchedule)) + exception.Throw(ErrBadConfig.FormatError(pathUsageReportingSchedule)) } for i := 1; i < checks-1; i++ { delta := nextRuns[i].Sub(nextRuns[i-1]) if delta < time.Hour { - log.Tracef("%s must be at 1 hour apart", pathUsageReportingUsageSchedule) - exception.Throw(ErrBadConfig.FormatError(pathUsageReportingUsageSchedule)) + log.Tracef("%s must be at 1 hour apart", pathUsageReportingSchedule) + exception.Throw(ErrBadConfig.FormatError(pathUsageReportingSchedule)) } } } func (u *UsageReportingConfiguration) validateOffline() { - if _, err := cronexpr.Parse(u.Schedule); err != nil { - exception.Throw(ErrBadConfig.FormatError(pathUsageReportingSchedule)) + if _, err := cronexpr.Parse(u.OfflineSchedule); err != nil { + exception.Throw(ErrBadConfig.FormatError(pathUsageReportingOfflineSchedule)) } // reporting is offline, lets read the QA env vars @@ -184,8 +146,8 @@ func (u *UsageReportingConfiguration) validateOffline() { if _, err := cronexpr.Parse(val); err != nil { log.Tracef("Could not use %s (%s) it is not a proper cron schedule", qaUsageReportingScheduleEnvVar, val) } else { - log.Tracef("Using %s (%s) rather than the default (%s) for non-QA", qaUsageReportingScheduleEnvVar, val, u.Schedule) - u.Schedule = val + log.Tracef("Using %s (%s) rather than the default (%s) for non-QA", qaUsageReportingScheduleEnvVar, val, u.OfflineSchedule) + u.OfflineSchedule = val u.qaVars = true } } @@ -201,19 +163,19 @@ func (u *UsageReportingConfiguration) validateOffline() { } // Check the cron expressions - cron, err := cronexpr.Parse(u.Schedule) + cron, err := cronexpr.Parse(u.OfflineSchedule) if err != nil { - exception.Throw(ErrBadConfig.FormatError(pathUsageReportingSchedule)) + exception.Throw(ErrBadConfig.FormatError(pathUsageReportingOfflineSchedule)) } nextTwoRuns := cron.NextN(time.Now(), 2) if len(nextTwoRuns) != 2 { - exception.Throw(ErrBadConfig.FormatError(pathUsageReportingSchedule)) + exception.Throw(ErrBadConfig.FormatError(pathUsageReportingOfflineSchedule)) } u.reportGranularity = int(nextTwoRuns[1].Sub(nextTwoRuns[0]).Milliseconds()) // if no QA env vars are set then validate the schedule is at least hourly if nextTwoRuns[1].Sub(nextTwoRuns[0]) < time.Hour && !u.qaVars { - exception.Throw(ErrBadConfig.FormatError(pathUsageReportingSchedule)) + exception.Throw(ErrBadConfig.FormatError(pathUsageReportingOfflineSchedule)) } } @@ -222,34 +184,24 @@ func (u *UsageReportingConfiguration) GetURL() string { return u.URL } -// CanPublishUsage - Returns the publish boolean -func (u *UsageReportingConfiguration) CanPublishUsage() bool { +// CanPublish - Returns the publish boolean +func (u *UsageReportingConfiguration) CanPublish() bool { return u.Publish } -// CanPublishMetric - Returns the publish metric boolean -func (u *UsageReportingConfiguration) CanPublishMetric() bool { - return u.PublishMetric -} - -// GetInterval - Returns the publish interval -func (u *UsageReportingConfiguration) GetInterval() time.Duration { - return u.Interval -} - // IsOfflineMode - Returns the offline boolean func (u *UsageReportingConfiguration) IsOfflineMode() bool { return u.Offline } // GetSchedule - Returns the schedule string -func (u *UsageReportingConfiguration) GetSchedule() string { - return u.Schedule +func (u *UsageReportingConfiguration) GetOfflineSchedule() string { + return u.OfflineSchedule } -// GetUsageSchedule - Returns the schedule string for publishing reports -func (u *UsageReportingConfiguration) GetUsageSchedule() string { - return u.UsageSchedule +// GetSchedule - Returns the schedule string for publishing reports +func (u *UsageReportingConfiguration) GetSchedule() string { + return u.Schedule } // GetReportSchedule - Returns the offline schedule string for creating reports @@ -259,9 +211,6 @@ func (u *UsageReportingConfiguration) GetReportSchedule() string { // GetReportGranularity - Returns the granularity used in the offline reports func (u *UsageReportingConfiguration) GetReportGranularity() int { - if u.reportGranularity == 0 { - return int(u.Interval.Milliseconds()) - } return u.reportGranularity } @@ -273,11 +222,9 @@ func (u *UsageReportingConfiguration) UsingQAVars() bool { // AddUsageReportingProperties - Adds the command properties needed for Usage Reporting Settings func AddUsageReportingProperties(props properties.Properties) { props.AddBoolProperty(pathUsageReportingPublish, true, "Indicates if the agent can publish usage events to Amplify platform. Default to true") - props.AddBoolProperty(pathUsageReportingPublishMetric, true, "Indicates if the agent can publish metric events to Amplify platform. Default to true") - props.AddDurationProperty(pathUsageReportingInterval, 15*time.Minute, "The time interval at which usage and metric events will be generated", properties.WithLowerLimit(5*time.Minute)) - props.AddStringProperty(pathUsageReportingUsageSchedule, "@daily", "The schedule at usage events are sent to the platform") + props.AddStringProperty(pathUsageReportingSchedule, "@daily", "The schedule at usage events are sent to the platform") props.AddBoolProperty(pathUsageReportingOffline, false, "Turn this on to save the usage events to disk for manual upload") - props.AddStringProperty(pathUsageReportingSchedule, "@hourly", "The schedule at which usage events are generated, for offline mode only") + props.AddStringProperty(pathUsageReportingOfflineSchedule, "@hourly", "The schedule at which usage events are generated, for offline mode only") } // ParseUsageReportingConfig - Parses the Usage Reporting Config values from the command line @@ -288,11 +235,9 @@ func ParseUsageReportingConfig(props properties.Properties) UsageReportingConfig // update the config cfg.Publish = props.BoolPropertyValue(pathUsageReportingPublish) - cfg.PublishMetric = props.BoolPropertyValue(pathUsageReportingPublishMetric) - cfg.Interval = props.DurationPropertyValue(pathUsageReportingInterval) - cfg.UsageSchedule = props.StringPropertyValue(pathUsageReportingUsageSchedule) - cfg.Offline = props.BoolPropertyValue(pathUsageReportingOffline) cfg.Schedule = props.StringPropertyValue(pathUsageReportingSchedule) + cfg.Offline = props.BoolPropertyValue(pathUsageReportingOffline) + cfg.OfflineSchedule = props.StringPropertyValue(pathUsageReportingOfflineSchedule) return cfg } diff --git a/pkg/config/usagereportingconfig_test.go b/pkg/config/usagereportingconfig_test.go index bb4d8aff4..cf4b7f14f 100644 --- a/pkg/config/usagereportingconfig_test.go +++ b/pkg/config/usagereportingconfig_test.go @@ -4,7 +4,6 @@ import ( "os" "strconv" "testing" - "time" "github.com/Axway/agent-sdk/pkg/cmd/properties" "github.com/Axway/agent-sdk/pkg/util/exception" @@ -27,11 +26,10 @@ func validateUsageReporting(cfg UsageReportingConfig) (err error) { type expected struct { url string publish bool - metric bool subscriptionMetric bool - interval time.Duration - offline bool schedule string + offline bool + offlineSchedule string reportSchedule string granularity int qaVars bool @@ -39,23 +37,21 @@ type expected struct { var defaultExpected = expected{ publish: true, - metric: true, subscriptionMetric: false, - interval: 15 * time.Minute, + schedule: "@daily", offline: false, - schedule: "@hourly", + offlineSchedule: "@hourly", reportSchedule: "@monthly", - granularity: int((15 * time.Minute).Milliseconds()), + granularity: 0, qaVars: false, } func validateconfig(t *testing.T, expVals expected, cfg UsageReportingConfig) { assert.Equal(t, expVals.url, cfg.GetURL()) - assert.Equal(t, expVals.publish, cfg.CanPublishUsage()) - assert.Equal(t, expVals.metric, cfg.CanPublishMetric()) - assert.Equal(t, expVals.interval, cfg.GetInterval()) - assert.Equal(t, expVals.offline, cfg.IsOfflineMode()) + assert.Equal(t, expVals.publish, cfg.CanPublish()) assert.Equal(t, expVals.schedule, cfg.GetSchedule()) + assert.Equal(t, expVals.offline, cfg.IsOfflineMode()) + assert.Equal(t, expVals.offlineSchedule, cfg.GetOfflineSchedule()) assert.Equal(t, expVals.reportSchedule, cfg.GetReportSchedule()) assert.Equal(t, expVals.granularity, cfg.GetReportGranularity()) assert.Equal(t, expVals.qaVars, cfg.UsingQAVars()) @@ -69,11 +65,7 @@ func TestUsageReportingConfigEnvVarMigration(t *testing.T) { props := properties.NewProperties(rootCmd) AddUsageReportingProperties(props) - // Test Interval old env vars - os.Setenv(oldUsageReportingIntervalEnvVar, "30m") expected := defaultExpected - expected.interval = 30 * time.Minute - expected.granularity = int((30 * time.Minute).Milliseconds()) cfg := ParseUsageReportingConfig(props) assert.NotNil(t, cfg) @@ -81,9 +73,6 @@ func TestUsageReportingConfigEnvVarMigration(t *testing.T) { assert.Nil(t, err) validateconfig(t, expected, cfg) - // Test Interval new env vars - os.Setenv(newUsageReportingIntervalEnvVar, defaultExpected.interval.String()) - expected = defaultExpected cfg = ParseUsageReportingConfig(props) assert.NotNil(t, cfg) @@ -115,7 +104,6 @@ func TestUsageReportingConfigEnvVarMigration(t *testing.T) { // Test PublishMetric old env vars os.Setenv(oldUsageReportingPublishMetricEnvVar, "true") expected = defaultExpected - expected.metric = true cfg = ParseUsageReportingConfig(props) assert.NotNil(t, cfg) @@ -123,9 +111,6 @@ func TestUsageReportingConfigEnvVarMigration(t *testing.T) { assert.Nil(t, err) validateconfig(t, expected, cfg) - // Test PublishMetric new env vars - os.Setenv(newUsageReportingPublishMetricEnvVar, strconv.FormatBool(defaultExpected.metric)) - expected = defaultExpected cfg = ParseUsageReportingConfig(props) assert.NotNil(t, cfg) @@ -152,26 +137,37 @@ func TestUsageReportingConfigProperties(t *testing.T) { validateconfig(t, defaultExpected, cfg) - // invalid Interval - currentInterval := cfg.GetInterval() - cfg.(*UsageReportingConfiguration).Interval = time.Millisecond + // invalid UsageSchedule + currentUsageSchedule := cfg.GetSchedule() + cfg.(*UsageReportingConfiguration).Schedule = "*/1511 * * * *" + err = validateUsageReporting(cfg) + assert.NotNil(t, err) + cfg.(*UsageReportingConfiguration).Schedule = "0,15,30,45,55 * * * *" err = validateUsageReporting(cfg) assert.NotNil(t, err) - cfg.(*UsageReportingConfiguration).Interval = currentInterval + cfg.(*UsageReportingConfiguration).Schedule = currentUsageSchedule - // invalid UsageSchedule - currentUsageSchedule := cfg.GetUsageSchedule() - cfg.(*UsageReportingConfiguration).UsageSchedule = "*/1511 * * * *" + // invalid old UsageSchedule override + os.Setenv(oldUsageReportingScheduleEnvVar, "*/1 * * *") err = validateUsageReporting(cfg) assert.NotNil(t, err) - cfg.(*UsageReportingConfiguration).UsageSchedule = "0,15,30,45,55 * * * *" + + // invalid old UsageSchedule override + os.Setenv(oldUsageReportingScheduleEnvVar, "*/1 * * * *") err = validateUsageReporting(cfg) assert.NotNil(t, err) - cfg.(*UsageReportingConfiguration).UsageSchedule = currentUsageSchedule + + // old UsageSchedule override + expected := defaultExpected + expected.schedule = "0 */5 * * *" + os.Setenv(oldUsageReportingScheduleEnvVar, "0 */5 * * *") + err = validateUsageReporting(cfg) + assert.Nil(t, err) + validateconfig(t, expected, cfg) // QA UsageSchedule override os.Setenv(qaUsageReportingUsageScheduleEnvVar, "*/1 * * * *") - cfg.(*UsageReportingConfiguration).UsageSchedule = "*/1 * * * *" + cfg.(*UsageReportingConfiguration).Schedule = "*/1 * * * *" err = validateUsageReporting(cfg) assert.Nil(t, err) @@ -181,15 +177,15 @@ func TestUsageReportingConfigProperties(t *testing.T) { assert.Nil(t, err) // invalid Schedule - currentSchedule := cfg.GetSchedule() - cfg.(*UsageReportingConfiguration).Schedule = "*/1511 * * * *" + currentSchedule := cfg.GetOfflineSchedule() + cfg.(*UsageReportingConfiguration).OfflineSchedule = "*/1511 * * * *" err = validateUsageReporting(cfg) assert.NotNil(t, err) - cfg.(*UsageReportingConfiguration).Schedule = currentSchedule + cfg.(*UsageReportingConfiguration).OfflineSchedule = currentSchedule // QA Schedule override os.Setenv(qaUsageReportingScheduleEnvVar, "*/1 * * * *") - cfg.(*UsageReportingConfiguration).Schedule = "*/1 * * * *" + cfg.(*UsageReportingConfiguration).OfflineSchedule = "*/1 * * * *" err = validateUsageReporting(cfg) assert.Nil(t, err) diff --git a/pkg/traceability/sampling/definitions.go b/pkg/traceability/sampling/definitions.go index 8c9c2ac07..1c714d173 100644 --- a/pkg/traceability/sampling/definitions.go +++ b/pkg/traceability/sampling/definitions.go @@ -5,7 +5,7 @@ package sampling const ( SampleKey = "sample" countMax = 100 - defaultSamplingRate = 1 + defaultSamplingRate = 0 maximumSamplingRate = 10 globalCounter = "global" ) diff --git a/pkg/traceability/sampling/globalsampling.go b/pkg/traceability/sampling/globalsampling.go index 5e179c6a6..fdd30a3cf 100644 --- a/pkg/traceability/sampling/globalsampling.go +++ b/pkg/traceability/sampling/globalsampling.go @@ -25,7 +25,7 @@ type Sampling struct { Percentage float64 `config:"percentage"` PerAPI bool `config:"per_api"` PerSub bool `config:"per_subscription"` - ReportAllErrors bool `config:"reportAllErrors" yaml:"reportAllErrors"` + OnlyErrors bool `config:"onlyErrors" yaml:"onlyErrors"` countMax int shouldSampleMax int } @@ -36,7 +36,7 @@ func DefaultConfig() Sampling { Percentage: defaultSamplingRate, PerAPI: true, PerSub: true, - ReportAllErrors: true, + OnlyErrors: false, countMax: countMax, shouldSampleMax: defaultSamplingRate, } @@ -80,10 +80,10 @@ func SetupSampling(cfg Sampling, offlineMode bool) error { if offlineMode { cfg = Sampling{ - Percentage: 0, - PerAPI: false, - PerSub: false, - ReportAllErrors: false, + Percentage: 0, + PerAPI: false, + PerSub: false, + OnlyErrors: false, } } else { cfg.Percentage, err = getSamplingPercentageConfig(cfg.Percentage) diff --git a/pkg/traceability/sampling/sample.go b/pkg/traceability/sampling/sample.go index 83c1d32bd..01c91f6af 100644 --- a/pkg/traceability/sampling/sample.go +++ b/pkg/traceability/sampling/sample.go @@ -17,9 +17,9 @@ type sample struct { // ShouldSampleTransaction - receives the transaction details and returns true to sample it false to not func (s *sample) ShouldSampleTransaction(details TransactionDetails) bool { hasFailedStatus := details.Status == "Failure" - // sample the transaction if reportAllErrors is set to `true` and the transaction summary's status is an error - if hasFailedStatus && s.config.ReportAllErrors { - return true + // sample only failed transaction if OnlyErrors is set to `true` and the transaction summary's status is an error + if !hasFailedStatus && s.config.OnlyErrors { + return false } sampleGlobal := s.shouldSampleWithCounter(globalCounter) diff --git a/pkg/traceability/sampling/sampling_test.go b/pkg/traceability/sampling/sampling_test.go index 23c991894..93f62c1f8 100644 --- a/pkg/traceability/sampling/sampling_test.go +++ b/pkg/traceability/sampling/sampling_test.go @@ -30,7 +30,7 @@ func TestSamplingConfig(t *testing.T) { errExpected: false, config: DefaultConfig(), expectedConfig: Sampling{ - Percentage: 1, + Percentage: 0, }, }, { @@ -94,8 +94,8 @@ func TestSamplingConfig(t *testing.T) { name: "Good Config, Report All Errors", errExpected: false, config: Sampling{ - Percentage: 10, - ReportAllErrors: true, + Percentage: 10, + OnlyErrors: true, }, expectedConfig: Sampling{ Percentage: 10, @@ -127,19 +127,22 @@ func TestSamplingConfig(t *testing.T) { } func TestShouldSample(t *testing.T) { + type transactionCount struct { + successCount int + errorCount int + } testCases := []struct { - name string - apiTransactions map[string]int - testTransactions int - expectedSampled int - config Sampling - subIDs map[string]string + name string + apiTransactions map[string]transactionCount + expectedSampled int + config Sampling + subIDs map[string]string }{ { name: "Maximum Transactions", - apiTransactions: map[string]int{ - "id1": 1000, - "id2": 1000, + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 1000}, + "id2": {successCount: 1000}, }, expectedSampled: 200, config: Sampling{ @@ -149,20 +152,20 @@ func TestShouldSample(t *testing.T) { }, { name: "Default config transactions", - apiTransactions: map[string]int{ - "id1": 1000, - "id2": 1000, + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 1000}, + "id2": {successCount: 1000}, }, - expectedSampled: 20, + expectedSampled: 0, config: DefaultConfig(), }, { name: "5% of Transactions when per api is disabled", - apiTransactions: map[string]int{ - "id1": 50, - "id2": 50, - "id3": 50, - "id4": 50, + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 50}, + "id2": {successCount: 50}, + "id3": {successCount: 50}, + "id4": {successCount: 50}, }, // Total = 200 expectedSampled: 10, config: Sampling{ @@ -172,9 +175,9 @@ func TestShouldSample(t *testing.T) { }, { name: "10% of Transactions when per api is disabled", - apiTransactions: map[string]int{ - "id1": 1000, - "id2": 1000, + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 1000}, + "id2": {successCount: 1000}, }, expectedSampled: 200, config: Sampling{ @@ -184,8 +187,8 @@ func TestShouldSample(t *testing.T) { }, { name: "0.55% of Transactions when per api is disabled", - apiTransactions: map[string]int{ - "id1": 10000, + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 10000}, }, expectedSampled: 55, config: Sampling{ @@ -195,10 +198,10 @@ func TestShouldSample(t *testing.T) { }, { name: "9.99% of Transactions when per api is disabled", - apiTransactions: map[string]int{ - "id1": 10000, - "id2": 10000, - "id3": 10000, + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 10000}, + "id2": {successCount: 10000}, + "id3": {successCount: 10000}, }, expectedSampled: 30000 * 999 / 10000, config: Sampling{ @@ -208,8 +211,8 @@ func TestShouldSample(t *testing.T) { }, { name: "0.0006% of Transactions when per api is disabled", - apiTransactions: map[string]int{ - "id1": 2000000, + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 2000000}, }, expectedSampled: 12, config: Sampling{ @@ -219,9 +222,9 @@ func TestShouldSample(t *testing.T) { }, { name: "1% of Transactions when per api is disabled", - apiTransactions: map[string]int{ - "id1": 1000, - "id2": 1000, + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 1000}, + "id2": {successCount: 1000}, }, expectedSampled: 20, config: Sampling{ @@ -231,9 +234,9 @@ func TestShouldSample(t *testing.T) { }, { name: "0% of Transactions when per api is disabled", - apiTransactions: map[string]int{ - "id1": 1000, - "id2": 1000, + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 1000}, + "id2": {successCount: 1000}, }, expectedSampled: 0, config: Sampling{ @@ -243,11 +246,11 @@ func TestShouldSample(t *testing.T) { }, { name: "5% per API of Transactions when per api is enabled", - apiTransactions: map[string]int{ - "id1": 50, // expect 50 - "id2": 50, // expect 50 - "id3": 50, // expect 50 - "id4": 50, // expect 50 + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 50}, // expect 50 + "id2": {successCount: 50}, // expect 50 + "id3": {successCount: 50}, // expect 50 + "id4": {successCount: 50}, // expect 50 }, expectedSampled: 20, config: Sampling{ @@ -257,11 +260,11 @@ func TestShouldSample(t *testing.T) { }, { name: "5% of subscription transactions when per api and per sub are enabled", - apiTransactions: map[string]int{ - "id1": 50, // expect 50 - "id2": 50, // expect 50 - "id3": 50, // expect 50 - "id4": 50, // expect 50 + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 50}, // expect 50 + "id2": {successCount: 50}, // expect 50 + "id3": {successCount: 50}, // expect 50 + "id4": {successCount: 50}, // expect 50 }, subIDs: map[string]string{ "id1": "sub1", @@ -278,11 +281,11 @@ func TestShouldSample(t *testing.T) { }, { name: "5% of subscription transactions when per api is disabled and per sub is enabled", - apiTransactions: map[string]int{ - "id1": 50, // expect 50 - "id2": 50, // expect 50 - "id3": 50, // expect 50 - "id4": 50, // expect 50 + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 50}, // expect 50 + "id2": {successCount: 50}, // expect 50 + "id3": {successCount: 50}, // expect 50 + "id4": {successCount: 50}, // expect 50 }, subIDs: map[string]string{ "id1": "sub1", @@ -299,11 +302,11 @@ func TestShouldSample(t *testing.T) { }, { name: "5% of per API transactions when per api and per sub are enabled, but no subID is found", - apiTransactions: map[string]int{ - "id1": 50, // expect 50 - "id2": 50, // expect 50 - "id3": 50, // expect 50 - "id4": 50, // expect 50 + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 50}, // expect 50 + "id2": {successCount: 50}, // expect 50 + "id3": {successCount: 50}, // expect 50 + "id4": {successCount: 50}, // expect 50 }, subIDs: map[string]string{}, expectedSampled: 20, @@ -313,6 +316,29 @@ func TestShouldSample(t *testing.T) { PerSub: true, }, }, + { + name: "only errors to be sampled", + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 500, errorCount: 500}, + "id2": {successCount: 500, errorCount: 500}, + }, + expectedSampled: 100, + config: Sampling{ + Percentage: 10, + OnlyErrors: true, + }, + }, + { + name: "errors and success to be sampled", + apiTransactions: map[string]transactionCount{ + "id1": {successCount: 500, errorCount: 500}, + "id2": {successCount: 500, errorCount: 500}, + }, + expectedSampled: 200, + config: Sampling{ + Percentage: 10, + }, + }, } for _, test := range testCases { @@ -335,11 +361,11 @@ func TestShouldSample(t *testing.T) { subID = test.subIDs[apiID] } - go func(wg *sync.WaitGroup, id, subID string, calls int) { + go func(wg *sync.WaitGroup, id, subID string, calls transactionCount) { defer wg.Done() - for i := 0; i < calls; i++ { + sampleFunc := func(id, subID string, status string) { testDetails := TransactionDetails{ - Status: "Success", // this does not matter at the moment + Status: status, APIID: id, SubID: subID, } @@ -351,6 +377,12 @@ func TestShouldSample(t *testing.T) { } assert.Nil(t, err) } + for i := 0; i < calls.successCount; i++ { + sampleFunc(id, subID, "Success") + } + for i := 0; i < calls.errorCount; i++ { + sampleFunc(id, subID, "Failure") + } }(&waitGroup, apiID, subID, numCalls) } diff --git a/pkg/traceability/traceability_test.go b/pkg/traceability/traceability_test.go index 8b2ea0d15..47bc457a1 100644 --- a/pkg/traceability/traceability_test.go +++ b/pkg/traceability/traceability_test.go @@ -50,7 +50,7 @@ func createCentralCfg(url, env string) *config.CentralConfiguration { authCfg.ClientID = "serviceaccount_1234" authCfg.PrivateKey = "../transaction/testdata/private_key.pem" authCfg.PublicKey = "../transaction/testdata/public_key" - cfg.GetUsageReportingConfig().(*config.UsageReportingConfiguration).Interval = 30 * time.Second + cfg.GetMetricReportingConfig().(*config.MetricReportingConfiguration).Schedule = "* * * * *" // every minute cfg.GetUsageReportingConfig().(*config.UsageReportingConfiguration).Offline = false return cfg } diff --git a/pkg/transaction/eventgenerator_test.go b/pkg/transaction/eventgenerator_test.go index 9cad75161..d0cae6de2 100644 --- a/pkg/transaction/eventgenerator_test.go +++ b/pkg/transaction/eventgenerator_test.go @@ -32,6 +32,7 @@ func createMapperTestConfig(authURL, tenantID, apicDeployment, envName, envID st APIServerVersion: "v1alpha1", SubscriptionConfiguration: corecfg.NewSubscriptionConfig(), UsageReporting: corecfg.NewUsageReporting("https://platform.xxx.com"), + MetricReporting: corecfg.NewMetricReporting(), ReportActivityFrequency: 2 * time.Minute, APIValidationCronSchedule: "@daily", ClientTimeout: 1 * time.Minute, diff --git a/pkg/transaction/logeventbuilder.go b/pkg/transaction/logeventbuilder.go index 72c49e5b6..3df2c3517 100644 --- a/pkg/transaction/logeventbuilder.go +++ b/pkg/transaction/logeventbuilder.go @@ -150,7 +150,7 @@ func NewTransactionSummaryBuilder() SummaryBuilder { txSummaryBuilder.logEvent.EnvironmentID = cfg.GetEnvironmentID() txSummaryBuilder.logEvent.APICDeployment = cfg.GetAPICDeployment() txSummaryBuilder.logEvent.TransactionSummary.IsInMetricEvent = - cfg.GetUsageReportingConfig().CanPublishMetric() + cfg.GetMetricReportingConfig().CanPublish() } return txSummaryBuilder } diff --git a/pkg/transaction/metric/cachestorage.go b/pkg/transaction/metric/cachestorage.go index 9a53aeea7..cd31e63e3 100644 --- a/pkg/transaction/metric/cachestorage.go +++ b/pkg/transaction/metric/cachestorage.go @@ -112,7 +112,7 @@ func (c *cacheStorage) loadUsage(storageCache cache.Cache) { } func (c *cacheStorage) updateUsage(usageCount int) { - if !c.isInitialized || !agent.GetCentralConfig().GetUsageReportingConfig().CanPublishUsage() { + if !c.isInitialized || !agent.GetCentralConfig().GetUsageReportingConfig().CanPublish() { return } @@ -125,7 +125,7 @@ func (c *cacheStorage) updateUsage(usageCount int) { func (c *cacheStorage) updateVolume(bytes int64) { if !c.isInitialized || !agent.GetCentralConfig().IsAxwayManaged() || - !agent.GetCentralConfig().GetUsageReportingConfig().CanPublishUsage() { + !agent.GetCentralConfig().GetUsageReportingConfig().CanPublish() { // NOT initialized or NOT axway managed or can NOT publish usage return } @@ -136,7 +136,7 @@ func (c *cacheStorage) updateVolume(bytes int64) { } func (c *cacheStorage) updateAppUsage(usageCount int, appID string) { - if !c.isInitialized || !agent.GetCentralConfig().GetUsageReportingConfig().CanPublishUsage() { + if !c.isInitialized || !agent.GetCentralConfig().GetUsageReportingConfig().CanPublish() { return } diff --git a/pkg/transaction/metric/metricscollector.go b/pkg/transaction/metric/metricscollector.go index b1efe5ce1..9c84a116e 100644 --- a/pkg/transaction/metric/metricscollector.go +++ b/pkg/transaction/metric/metricscollector.go @@ -78,6 +78,7 @@ type collector struct { usagePublisher *usagePublisher storage storageCache reports *usageReportCache + metricConfig config.MetricReportingConfig usageConfig config.UsageReportingConfig logger log.FieldLogger metricLogger log.FieldLogger @@ -165,6 +166,7 @@ func createMetricCollector() Collector { registry: metrics.NewRegistry(), metricMap: make(map[string]map[string]map[string]map[string]*APIMetric), publishItemQueue: make([]publishQueueItem, 0), + metricConfig: agent.GetCentralConfig().GetMetricReportingConfig(), usageConfig: agent.GetCentralConfig().GetUsageReportingConfig(), logger: logger, metricLogger: log.NewMetricFieldLogger(), @@ -179,9 +181,9 @@ func createMetricCollector() Collector { if util.IsNotTest() { var err error if !metricCollector.usageConfig.IsOfflineMode() { - metricCollector.jobID, err = jobs.RegisterIntervalJobWithName(metricCollector, metricCollector.usageConfig.GetInterval(), "Metric Collector") + metricCollector.jobID, err = jobs.RegisterScheduledJobWithName(metricCollector, metricCollector.metricConfig.GetSchedule(), "Metric Collector") } else { - metricCollector.jobID, err = jobs.RegisterScheduledJobWithName(metricCollector, metricCollector.usageConfig.GetSchedule(), "Metric Collector") + metricCollector.jobID, err = jobs.RegisterScheduledJobWithName(metricCollector, metricCollector.usageConfig.GetOfflineSchedule(), "Metric Collector") } if err != nil { panic(err) @@ -265,7 +267,7 @@ func (c *collector) AddMetricDetail(metricDetail Detail) { // AddMetricDetailSet - add metric details for several response codes and transactions func (c *collector) AddAPIMetricDetail(detail MetricDetail) { - if !c.usageConfig.CanPublishMetric() || c.usageConfig.IsOfflineMode() { + if !c.metricConfig.CanPublish() || c.usageConfig.IsOfflineMode() { return } @@ -362,7 +364,7 @@ func (c *collector) createMetric(detail TransactionContext) (*APIMetric, []strin } func (c *collector) createOrUpdateMetric(detail Detail) *APIMetric { - if !c.usageConfig.CanPublishMetric() || c.usageConfig.IsOfflineMode() { + if !c.metricConfig.CanPublish() || c.usageConfig.IsOfflineMode() { return nil // no need to update metrics with publish off } @@ -617,7 +619,7 @@ func (c *collector) generateEvents() { Info("no metric events generated as no transactions recorded") } - if c.usageConfig.CanPublishMetric() { + if c.metricConfig.CanPublish() { err := c.metricBatch.Publish() if err != nil { c.logger.WithError(err).Errorf("could not send metric event, data is kept and will be added to the next trigger interval") @@ -628,7 +630,7 @@ func (c *collector) generateEvents() { func (c *collector) processRegistry(name string, metric interface{}) { switch { case name == transactionCountMetric: - if c.usageConfig.CanPublishUsage() { + if c.usageConfig.CanPublish() { c.generateUsageEvent(c.orgGUID) } else { c.logger.Info("Publishing the usage event is turned off") @@ -669,6 +671,11 @@ func (c *collector) generateUsageEvent(orgGUID string) { } granularity := c.usageConfig.GetReportGranularity() + // for offline usage reporting granularity computed with offline schedule + if granularity == 0 { + granularity = c.metricConfig.GetReportGranularity() + } + reportTime := c.usageStartTime.Format(ISO8601) if c.usageConfig.IsOfflineMode() { reportTime = c.usageEndTime.Add(time.Duration(-1*granularity) * time.Millisecond).Format(ISO8601) diff --git a/pkg/transaction/metric/metricscollector_test.go b/pkg/transaction/metric/metricscollector_test.go index a80cb2628..6fe7ee76a 100644 --- a/pkg/transaction/metric/metricscollector_test.go +++ b/pkg/transaction/metric/metricscollector_test.go @@ -62,7 +62,8 @@ func createCentralCfg(url, env string) *config.CentralConfiguration { authCfg.PublicKey = "../../transaction/testdata/public_key" usgCfg := cfg.UsageReporting.(*config.UsageReportingConfiguration) usgCfg.Publish = true - usgCfg.PublishMetric = true + metricCfg := cfg.MetricReporting.(*config.MetricReportingConfiguration) + metricCfg.Publish = true return cfg } @@ -274,7 +275,7 @@ func TestMetricCollector(t *testing.T) { cfg := createCentralCfg(s.server.URL, "demo") cfg.UsageReporting.(*config.UsageReportingConfiguration).URL = s.server.URL + "/lighthouse" - cfg.UsageReporting.(*config.UsageReportingConfiguration).PublishMetric = true + cfg.MetricReporting.(*config.MetricReportingConfiguration).Publish = true cfg.SetEnvironmentID("267bd671-e5e2-4679-bcc3-bbe7b70f30fd") cmd.BuildDataPlaneType = "Azure" agent.Initialize(cfg) @@ -511,7 +512,7 @@ func TestConcurrentMetricCollectorEvents(t *testing.T) { cfg := createCentralCfg(s.server.URL, "demo") cfg.UsageReporting.(*config.UsageReportingConfiguration).URL = s.server.URL + "/lighthouse" - cfg.UsageReporting.(*config.UsageReportingConfiguration).PublishMetric = true + cfg.MetricReporting.(*config.MetricReportingConfiguration).Publish = true cfg.SetEnvironmentID("267bd671-e5e2-4679-bcc3-bbe7b70f30fd") cmd.BuildDataPlaneType = "Azure" agent.Initialize(cfg) @@ -584,7 +585,7 @@ func TestMetricCollectorUsageAggregation(t *testing.T) { cfg := createCentralCfg(s.server.URL, "demo") cfg.UsageReporting.(*config.UsageReportingConfiguration).URL = s.server.URL + "/lighthouse" - cfg.UsageReporting.(*config.UsageReportingConfiguration).PublishMetric = true + cfg.MetricReporting.(*config.MetricReportingConfiguration).Publish = true cfg.SetEnvironmentID("267bd671-e5e2-4679-bcc3-bbe7b70f30fd") cmd.BuildDataPlaneType = "Azure" agent.Initialize(cfg) diff --git a/pkg/transaction/metric/reportcache.go b/pkg/transaction/metric/reportcache.go index bb6604459..b1e04463e 100644 --- a/pkg/transaction/metric/reportcache.go +++ b/pkg/transaction/metric/reportcache.go @@ -79,7 +79,7 @@ func (c *usageReportCache) getEvents() UsageEvent { // loadEvents - locks the cache before getting the events func (c *usageReportCache) loadEvents() UsageEvent { - if !agent.GetCentralConfig().GetUsageReportingConfig().CanPublishUsage() { + if !agent.GetCentralConfig().GetUsageReportingConfig().CanPublish() { return UsageEvent{Report: map[string]UsageReport{}} } c.reportCacheLock.Lock() @@ -100,7 +100,7 @@ func (c *usageReportCache) setEvents(lighthouseEvent UsageEvent) { // updateEvents - locks the cache before setting the new light house events in the cache func (c *usageReportCache) updateEvents(lighthouseEvent UsageEvent) { - if !c.isInitialized || !agent.GetCentralConfig().GetUsageReportingConfig().CanPublishUsage() { + if !c.isInitialized || !agent.GetCentralConfig().GetUsageReportingConfig().CanPublish() { return } diff --git a/pkg/transaction/metric/usagepublisher.go b/pkg/transaction/metric/usagepublisher.go index 0f456f7e0..67f686895 100644 --- a/pkg/transaction/metric/usagepublisher.go +++ b/pkg/transaction/metric/usagepublisher.go @@ -193,7 +193,7 @@ func (c *usagePublisher) registerReportJob() { return // skip setting up the job in test } - schedule := agent.GetCentralConfig().GetUsageReportingConfig().GetUsageSchedule() + schedule := agent.GetCentralConfig().GetUsageReportingConfig().GetSchedule() if agent.GetCentralConfig().GetUsageReportingConfig().IsOfflineMode() { schedule = agent.GetCentralConfig().GetUsageReportingConfig().GetReportSchedule() }