Add Wildcard Refresh Interval (#276)

* confgenerator: add wildcard_refresh_interval

Adds new field wildcard_refresh_interval to the logging files receiver
in the ops-agent config.

* confgenerator: added 3rd party receiver tests

* confgenerator: removed unused casting case

* confgenerator: change validator to multipleof_time

Use a different strategy for the validator that is more scalable than
the original `whole_time_denomination` implementation; take a
time.Duration and see if it is a multiple of some other duration.

* confgenerator: fixed copyright comment

confgenerator/config.go

@@ -90,6 +90,10 @@ func (ve validationError) Error() string {
 		return fmt.Sprintf("%q must end with %q", ve.Field(), ve.Param())
 	case "ip":
 		return fmt.Sprintf("%q must be an IP address", ve.Field())
+	case "min":
+		return fmt.Sprintf("%q must be a minimum of %s", ve.Field(), ve.Param())
+	case "multipleof_time":
+		return fmt.Sprintf("%q must be a multiple of %s", ve.Field(), ve.Param())
 	case "oneof":
 		return fmt.Sprintf("%q must be one of [%s]", ve.Field(), ve.Param())
 	case "required":
@@ -137,7 +141,7 @@ func newValidator() *validator.Validate {
 	v.RegisterValidationCtx("platform", func(ctx context.Context, fl validator.FieldLevel) bool {
 		return ctx.Value(platformKey) == fl.Param()
 	})
-	// duration validates that the value is a valid durationa and >= the parameter
+	// duration validates that the value is a valid duration and >= the parameter
 	v.RegisterValidation("duration", func(fl validator.FieldLevel) bool {
 		t, err := time.ParseDuration(fl.Field().String())
 		if err != nil {
@@ -149,6 +153,18 @@ func newValidator() *validator.Validate {
 		}
 		return t >= tmin
 	})
+	// multipleof_time validates that the value duration is a multiple of the parameter
+	v.RegisterValidation("multipleof_time", func(fl validator.FieldLevel) bool {
+		t, ok := fl.Field().Interface().(time.Duration)
+		if !ok {
+			panic(fmt.Sprintf("multipleof_time: could not convert %s to time duration", fl.Field().String()))
+		}
+		tfactor, err := time.ParseDuration(fl.Param())
+		if err != nil {
+			panic(fmt.Sprintf("multipleof_time: could not convert %s to time duration", fl.Param()))
+		}
+		return t%tfactor == 0
+	})
 	return v
 }
 

confgenerator/logging_receivers.go

@@ -17,7 +17,9 @@ package confgenerator
 import (
 	"fmt"
 	"path"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit"
 )
@@ -33,8 +35,9 @@ func DBPath(tag string) string {
 type LoggingReceiverFiles struct {
 	ConfigComponent `yaml:",inline"`
 	// TODO: Use LoggingReceiverFilesMixin after figuring out the validation story.
-	IncludePaths []string `yaml:"include_paths,omitempty" validate:"required"`
-	ExcludePaths []string `yaml:"exclude_paths,omitempty"`
+	IncludePaths            []string       `yaml:"include_paths,omitempty" validate:"required"`
+	ExcludePaths            []string       `yaml:"exclude_paths,omitempty"`
+	WildcardRefreshInterval *time.Duration `yaml:"wildcard_refresh_interval,omitempty" validate:"omitempty,min=1s,multipleof_time=1s"`
 }
 
 func (r LoggingReceiverFiles) Type() string {
@@ -43,14 +46,16 @@ func (r LoggingReceiverFiles) Type() string {
 
 func (r LoggingReceiverFiles) Components(tag string) []fluentbit.Component {
 	return LoggingReceiverFilesMixin{
-		IncludePaths: r.IncludePaths,
-		ExcludePaths: r.ExcludePaths,
+		IncludePaths:            r.IncludePaths,
+		ExcludePaths:            r.ExcludePaths,
+		WildcardRefreshInterval: r.WildcardRefreshInterval,
 	}.Components(tag)
 }
 
 type LoggingReceiverFilesMixin struct {
-	IncludePaths []string `yaml:"include_paths,omitempty"`
-	ExcludePaths []string `yaml:"exclude_paths,omitempty"`
+	IncludePaths            []string       `yaml:"include_paths,omitempty"`
+	ExcludePaths            []string       `yaml:"exclude_paths,omitempty"`
+	WildcardRefreshInterval *time.Duration `yaml:"wildcard_refresh_interval,omitempty" validate:"omitempty,min=1s,multipleof_time=1s"`
 }
 
 func (r LoggingReceiverFilesMixin) Components(tag string) []fluentbit.Component {
@@ -92,6 +97,10 @@ func (r LoggingReceiverFilesMixin) Components(tag string) []fluentbit.Component
 		// TODO: Escaping?
 		config["Exclude_Path"] = strings.Join(r.ExcludePaths, ",")
 	}
+	if r.WildcardRefreshInterval != nil {
+		refreshIntervalSeconds := int(r.WildcardRefreshInterval.Seconds())
+		config["Refresh_Interval"] = strconv.Itoa(refreshIntervalSeconds)
+	}
 	return []fluentbit.Component{{
 		Kind:   "INPUT",
 		Config: config,

confgenerator/testdata/invalid/linux/logging-receiver_files_refresh_interval_invalid_duration/golden_error

@@ -0,0 +1 @@
+cannot unmarshal uint64 into Go struct field UnifiedConfig.Logging of type time.Duration

confgenerator/testdata/invalid/linux/logging-receiver_files_refresh_interval_invalid_duration/input.yaml

@@ -0,0 +1,26 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+logging:
+  receivers:
+    log_source_id1:
+      type: files
+      include_paths:
+      - /path/to/log/1/*
+      wildcard_refresh_interval: 30
+  service:
+    pipelines:
+      pipeline1:
+        receivers:
+        - log_source_id1

confgenerator/testdata/invalid/linux/logging-receiver_files_refresh_interval_not_whole_seconds/golden_error

@@ -0,0 +1,9 @@
+[21:34] "wildcard_refresh_interval" must be a multiple of 1s
+  18 |       type: files
+  19 |       include_paths:
+  20 |       - /path/to/log/1/*
+> 21 |       wildcard_refresh_interval: 3500ms
+                                        ^
+  22 |   service:
+  23 |     pipelines:
+  24 |       pipeline1:

confgenerator/testdata/invalid/linux/logging-receiver_files_refresh_interval_not_whole_seconds/input.yaml

@@ -0,0 +1,26 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+logging:
+  receivers:
+    log_source_id1:
+      type: files
+      include_paths:
+      - /path/to/log/1/*
+      wildcard_refresh_interval: 3500ms
+  service:
+    pipelines:
+      pipeline1:
+        receivers:
+        - log_source_id1

confgenerator/testdata/invalid/linux/logging-receiver_files_refresh_interval_too_low/golden_error

@@ -0,0 +1,9 @@
+[21:34] "wildcard_refresh_interval" must be a minimum of 1s
+  18 |       type: files
+  19 |       include_paths:
+  20 |       - /path/to/log/1/*
+> 21 |       wildcard_refresh_interval: 0s
+                                        ^
+  22 |   service:
+  23 |     pipelines:
+  24 |       pipeline1:

confgenerator/testdata/invalid/linux/logging-receiver_files_refresh_interval_too_low/input.yaml

@@ -0,0 +1,26 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+logging:
+  receivers:
+    log_source_id1:
+      type: files
+      include_paths:
+      - /path/to/log/1/*
+      wildcard_refresh_interval: 0s
+  service:
+    pipelines:
+      pipeline1:
+        receivers:
+        - log_source_id1