diff --git a/config/base/env/config b/config/base/env/config index a671cbd6e..91ec5e64e 100644 --- a/config/base/env/config +++ b/config/base/env/config @@ -42,5 +42,6 @@ LOGGING_PLUGIN_TOKEN_PATH=/var/run/secrets/kubernetes.io/serviceaccount/token LOGGING_PLUGIN_NAMESPACE_KEY=kubernetes_namespace_name LOGGING_PLUGIN_STATIC_LABELS='log_type=application' LOGGING_PLUGIN_CA_CERT= +LOGGING_PLUGIN_QUERY_LIMIT=1700 LOGGING_PLUGIN_TLS_VERIFICATION_DISABLE= LOGGING_PLUGIN_FORWARDER_DELAY_DURATION=10 diff --git a/docs/logging-support.md b/docs/logging-support.md index 33c753756..cd79641ef 100644 --- a/docs/logging-support.md +++ b/docs/logging-support.md @@ -28,3 +28,4 @@ These are the common configuration options for all third party logging APIs. - `LOGGING_PLUGIN_CA_CERT`: The CA certificate to use for the third party logging API. This should ideally be passed as environment variable in the deployment spec of the results-api pod. (optional) - `LOGGING_PLUGIN_TLS_VERIFICATION_DISABLE`: Set to `true` to disable TLS verification for the third party logging API. (optional) - `LOGGING_PLUGIN_FORWARDER_DELAY_DURATION`: This is the max duration in minutes taken by third party logging system to forward and store the logs after completion of taskrun and pipelinerun. This is used to search between start time of runs and completion plus buffer duration. +- `LOGGING_PLUGIN_QUERY_LIMIT`: Sets the query limit for Third Party Logging API if logging backend has a limit on number of log lines returned. diff --git a/pkg/api/server/config/config.go b/pkg/api/server/config/config.go index 322d9fdfb..652eebf88 100644 --- a/pkg/api/server/config/config.go +++ b/pkg/api/server/config/config.go @@ -58,6 +58,7 @@ type Config struct { LOGGING_PLUGIN_TOKEN_PATH string `mapstructure:"LOGGING_PLUGIN_TOKEN_PATH"` LOGGING_PLUGIN_PROXY_PATH string `mapstructure:"LOGGING_PLUGIN_PROXY_PATH"` LOGGING_PLUGIN_CA_CERT string `mapstructure:"LOGGING_PLUGIN_CA_CERT"` + LOGGING_PLUGIN_QUERY_LIMIT uint `mapstructure:"LOGGING_PLUGIN_QUERY_LIMIT"` LOGGING_PLUGIN_TLS_VERIFICATION_DISABLE bool `mapstructure:"LOGGING_PLUGIN_TLS_VERIFICATION_DISABLE"` LOGGING_PLUGIN_FORWARDER_DELAY_DURATION uint `mapstructure:"LOGGING_PLUGIN_FORWARDER_DELAY_DURATION"` } diff --git a/pkg/api/server/v1alpha2/plugin_logs.go b/pkg/api/server/v1alpha2/plugin_logs.go index 699fb1ff6..bf4f0fee4 100644 --- a/pkg/api/server/v1alpha2/plugin_logs.go +++ b/pkg/api/server/v1alpha2/plugin_logs.go @@ -145,6 +145,7 @@ func (s *LogPluginServer) getLokiLogs(writer *logs.BufferedLog, parent string, r parameters.Add("query", `{ `+s.staticLabels+s.config.LOGGING_PLUGIN_NAMESPACE_KEY+`="`+parent+`" }|json uid="`+uidKey+`", message="message" |uid="`+rec.Name+`"| line_format "{{.message}}"`) parameters.Add("end", endTime) parameters.Add("start", startTime) + parameters.Add("limit", strconv.Itoa(int(s.queryLimit))) URL.RawQuery = parameters.Encode() s.logger.Debugf("loki request url:%s", URL.String()) diff --git a/pkg/api/server/v1alpha2/plugin_logs_test.go b/pkg/api/server/v1alpha2/plugin_logs_test.go index a8e838230..87f77096d 100644 --- a/pkg/api/server/v1alpha2/plugin_logs_test.go +++ b/pkg/api/server/v1alpha2/plugin_logs_test.go @@ -65,6 +65,7 @@ func TestLogPluginServer_GetLog(t *testing.T) { LOGGING_PLUGIN_TLS_VERIFICATION_DISABLE: true, LOGGING_PLUGIN_STATIC_LABELS: "namespace=\"foo\"", LOGGING_PLUGIN_NAMESPACE_KEY: "namespace", + LOGGING_PLUGIN_QUERY_LIMIT: 1500, }, logger.Get("info"), test.NewDB(t)) if err != nil { t.Fatalf("failed to create server: %v", err) diff --git a/pkg/api/server/v1alpha2/server.go b/pkg/api/server/v1alpha2/server.go index f0416d699..0239f49ea 100644 --- a/pkg/api/server/v1alpha2/server.go +++ b/pkg/api/server/v1alpha2/server.go @@ -83,6 +83,8 @@ type LogPluginServer struct { forwarderDelayDuration time.Duration + queryLimit uint + // TODO: In future add support for non Oauth support tokenSource oauth2.TokenSource } @@ -208,6 +210,7 @@ func (s *Server) createLogPluginServer() error { s.LogPluginServer.forwarderDelayDuration = time.Duration(s.config.LOGGING_PLUGIN_FORWARDER_DELAY_DURATION) * time.Minute s.LogPluginServer.tokenSource = transport.NewCachedFileTokenSource(s.config.LOGGING_PLUGIN_TOKEN_PATH) + s.LogPluginServer.queryLimit = s.config.LOGGING_PLUGIN_QUERY_LIMIT return nil }