Merge pull request #40 from calyptia/configload

Adds new configloader package

.golangci.yml

@@ -5,19 +5,24 @@ run:
   #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
   skip-dirs-use-default: true
 linters-settings:
+  funlen:
+    lines: 100
+    statements: 50
   goimports:
     local-prefixes: github.com/calyptia/cloud
   revive:
     min-confidence: 0.8
   gocyclo:
-    min-complexity: 15
+    min-complexity: 25
   govet:
     check-shadowing: true
   misspell:
     locale: US
   gomnd:
     ignored-numbers:
       - 2 #used for word splitting.
+  gocognit:
+    min-complexity: 54
   nolintlint:
     allow-leading-space: false # require machine-readable nolint directives (with no leading space)
     allow-unused: false # report any unused nolint directives
@@ -40,7 +45,6 @@ linters:
     - unused
     - varcheck
     - bodyclose
-    - depguard
     - dupl
     - exportloopref
     - forcetypeassert

configloader/doc.go

@@ -0,0 +1,22 @@
+// Package configloader provides functionality to load Calyptia configurations
+// from a set of specified configuration files. The main purpose of this package
+// is to provide a simplified and streamlined way to fetch configurations without
+// relying on environment variables.
+//
+// The package introduces a `Loader` type, which is responsible for loading and
+// parsing the configuration files. It leverages the FileReader interface to read
+// file contents, which allows for easy mocking in unit tests.
+//
+// Usage:
+//
+//	loader := configloader.NewLoader(nil) // uses the real file reader by default
+//	config := loader.LoadFromFiles("path/to/config1.yaml", "path/to/config2.conf")
+//
+//	if config != nil {
+//		fmt.Println("Loaded config:", config)
+//	}
+//
+// The package also supports multiple file formats, including `.conf`, `.ini`, `.yaml`,
+// and `.yml`. The first valid Calyptia configuration found among the provided files
+// will be returned. If none is found, the function returns nil.
+package configloader

configloader/go.sum

@@ -0,0 +1,20 @@
+github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0=
+github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
+github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
+github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/calyptia/go-fluentbit-config/v2 v2.1.0 h1:DWV0N4DO9vJX9I1Y3tdYi0P7A1xa+rqGsqLG06tUmEc=
+github.com/calyptia/go-fluentbit-config/v2 v2.1.0/go.mod h1:bmPxFuwb7//+pLPSvLbLxS/fYesxysQtiSksElDVszQ=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
+golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

configloader/loader.go

@@ -0,0 +1,154 @@
+package configloader
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	fluentbitconfig "github.com/calyptia/go-fluentbit-config/v2"
+)
+
+// CalyptiaConfig contains configurations related to Calyptia.
+type CalyptiaConfig struct {
+	Token string
+	URL   string
+	TLS   bool
+}
+
+// FileReader is an interface for reading files.
+type FileReader interface {
+	ReadFile(filename string) ([]byte, error)
+}
+
+// LoaderInterface is an interface for the configuration loader.
+type LoaderInterface interface {
+	LoadFromFiles(configFiles ...string) *CalyptiaConfig
+}
+
+// RealFileReader is a concrete implementation of FileReader.
+type RealFileReader struct{}
+
+// ReadFile reads the content of the given filename.
+func (r RealFileReader) ReadFile(filename string) ([]byte, error) {
+	//nolint:gosec //required filesystem access to read fixture data.
+	return os.ReadFile(filename)
+}
+
+// Loader is responsible for loading configurations.
+type Loader struct {
+	FileReader
+}
+
+// NewLoader initializes a new Loader with the given FileReader.
+func NewLoader(fr FileReader) *Loader {
+	if fr == nil {
+		fr = RealFileReader{}
+	}
+	return &Loader{fr}
+}
+
+// parseConfig reads the content of the given config path and parses it into a fluentbit configuration object.
+func (cl *Loader) parseConfig(path string) (*fluentbitconfig.Config, error) {
+	// Read the content from the given path using the FileReader
+	content, err := cl.FileReader.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("error reading file %s: %v", path, err)
+	}
+
+	// Determine the format based on the file extension
+	ext := filepath.Ext(path)
+	var format fluentbitconfig.Format
+	switch ext {
+	case ".conf", ".ini":
+		format = fluentbitconfig.FormatClassic
+	case ".yaml", ".yml":
+		format = fluentbitconfig.FormatYAML
+	default:
+		return nil, fmt.Errorf("unsupported file format for %s", path)
+	}
+
+	// Parse the content into a fluentbit configuration object
+	parsedConfig, err := fluentbitconfig.ParseAs(string(content), format)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing config from file %s: %v", path, err)
+	}
+
+	return &parsedConfig, nil
+}
+
+// LoadFromFiles loads the Calyptia configurations from the given list of files.
+// It goes through each file, parses it, and attempts to retrieve the Calyptia configuration.
+// The method returns the first valid Calyptia configuration it finds.
+func (cl *Loader) LoadFromFiles(configFiles ...string) *CalyptiaConfig {
+	calyptiaConfig := &CalyptiaConfig{
+		TLS: false, // default value
+	}
+
+	for _, path := range configFiles {
+		parsedConfig, err := cl.parseConfig(path)
+		if err != nil {
+			continue
+		}
+
+		// Attempt to find the calyptia plugin properties
+		plugin, found := parsedConfig.Customs.FindByID("calyptia.0")
+		if !found {
+			continue
+		}
+		// Extract calyptia_tls and determine whether it's "true" or "on"
+		value, exists := plugin.Properties.Get("calyptia_tls")
+		if exists {
+			tlsValue, ok := value.(string)
+			if ok {
+				tlsVal := strings.ToLower(tlsValue)
+				calyptiaConfig.TLS = tlsVal == "true" || tlsVal == "on"
+			} else {
+				calyptiaConfig.TLS = false
+			}
+		}
+
+		// Extract calyptia_host and construct full URL
+		value, exists = plugin.Properties.Get("calyptia_host")
+		if exists {
+			host, ok := value.(string)
+			if !ok {
+				host = "" // Set to empty string if type assertion fails
+			}
+			if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
+				if calyptiaConfig.TLS {
+					host = "https://" + host
+				} else {
+					host = "http://" + host
+				}
+			}
+			parsedURL, err := url.Parse(host)
+			if err == nil {
+				fullURL := parsedURL.Scheme + "://" + parsedURL.Hostname()
+				if !(calyptiaConfig.TLS && parsedURL.Port() == "443" || !calyptiaConfig.TLS && parsedURL.Port() == "80") && parsedURL.Port() != "" {
+					fullURL += ":" + parsedURL.Port()
+				}
+				calyptiaConfig.URL = fullURL
+			}
+		}
+
+		// Extract api_key
+		value, exists = plugin.Properties.Get("api_key")
+		if exists {
+			token, ok := value.(string)
+			if !ok {
+				token = "" // Set to empty string if type assertion fails
+			}
+			calyptiaConfig.Token = token
+		}
+
+		return calyptiaConfig
+	}
+	return nil
+}
+
+// NewDefaultLoader returns a new Loader instance with the default real file reader.
+func NewDefaultLoader() *Loader {
+	return &Loader{RealFileReader{}}
+}

configloader/loader_test.go

@@ -0,0 +1,72 @@
+package configloader
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigLoader(t *testing.T) {
+	tests := []struct {
+		name           string
+		fileName       string
+		expectedConfig *CalyptiaConfig
+		expectError    bool
+	}{
+		{
+			name:     "Basic YAML config provided",
+			fileName: "basic.yaml",
+			expectedConfig: &CalyptiaConfig{
+				URL:   "https://cloud-api.calyptia.com",
+				Token: "token",
+				TLS:   true,
+			},
+			expectError: false,
+		},
+		{
+			name:     "Basic YAML no TLS config provided",
+			fileName: "basic-no-tls.yaml",
+			expectedConfig: &CalyptiaConfig{
+				URL:   "http://cloud-api.calyptia.com",
+				Token: "token",
+				TLS:   false,
+			},
+			expectError: false,
+		},
+		{
+			name:     "Basic INI config provided",
+			fileName: "basic.ini",
+			expectedConfig: &CalyptiaConfig{
+				URL:   "https://cloud-api.calyptia.com",
+				Token: "token",
+				TLS:   true,
+			},
+			expectError: false,
+		},
+		{
+			name:           "Incorrect YAML config provided",
+			fileName:       "incorrect.yaml",
+			expectedConfig: nil,
+			expectError:    true,
+		},
+		{
+			name:           "Incorrect INI config provided",
+			fileName:       "incorrect.ini",
+			expectedConfig: nil,
+			expectError:    true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			loader := NewDefaultLoader()
+			config := loader.LoadFromFiles(filepath.Join("testdata", tt.fileName))
+			if tt.expectError {
+				assert.Nil(t, config)
+			} else {
+				assert.Equal(t, tt.expectedConfig, config)
+			}
+		})
+	}
+}

configloader/testdata/basic-no-tls.yaml

@@ -0,0 +1,19 @@
+HTTP_Server: "On"
+HTTP_Listen: 0.0.0.0
+HTTP_PORT: 2020
+Log_Level: debug
+plugins_file: /opt/calyptia-fluent-bit/etc/enterprise_plugins.conf
+Parsers_File: /config/parsers.conf
+customs:
+  - name: calyptia
+    calyptia_tls: "off"
+    api_key: token
+    pipeline_id: 774e7aed-cce1-4460-a160-1d24be8c37dd
+    add_label: pipeline_id 774e7aed-cce1-4460-a160-1d24be8c37dd
+    calyptia_host: cloud-api.calyptia.com
+pipeline:
+  inputs:
+    - name: fluentbit_metrics
+      tag: _calyptia_cloud
+      scrape_on_start: true
+      scrape_interval: 30

configloader/testdata/basic.ini

@@ -0,0 +1,24 @@
+[SERVICE]
+    HTTP_Server  On
+    HTTP_Listen  0.0.0.0
+    HTTP_PORT    2020
+    Log_Level debug
+
+[SERVICE]
+    plugins_file    /opt/calyptia-fluent-bit/etc/enterprise_plugins.conf
+    storage.path /data/storage
+    storage.backlog.mem_limit 5M
+    storage.max_chunks_up 128.000000
+    storage.sync normal
+[CUSTOM]
+    name                calyptia
+    api_key token
+    log_level           debug
+    pipeline_id       9c797f01-531a-4c16-8b54-efd576eb2ce9
+    add_label    pipeline_id 9c797f01-531a-4c16-8b54-efd576eb2ce9
+    add_label         pod_uid ${POD_UID}
+    calyptia_host       cloud-api.calyptia.com
+    calyptia_port       443
+    calyptia_tls        on
+    calyptia_tls.verify on
+

configloader/testdata/basic.yaml

@@ -0,0 +1,19 @@
+HTTP_Server: "On"
+HTTP_Listen: 0.0.0.0
+HTTP_PORT: 2020
+Log_Level: debug
+plugins_file: /opt/calyptia-fluent-bit/etc/enterprise_plugins.conf
+Parsers_File: /config/parsers.conf
+customs:
+  - name: calyptia
+    calyptia_tls: "on"
+    api_key: token
+    pipeline_id: 774e7aed-cce1-4460-a160-1d24be8c37dd
+    add_label: pipeline_id 774e7aed-cce1-4460-a160-1d24be8c37dd
+    calyptia_host: cloud-api.calyptia.com
+pipeline:
+  inputs:
+    - name: fluentbit_metrics
+      tag: _calyptia_cloud
+      scrape_on_start: true
+      scrape_interval: 30

configloader/testdata/incorrect.ini

@@ -0,0 +1,2 @@
+[SERVICE]
+wrongField example

configloader/testdata/incorrect.yaml

@@ -0,0 +1,2 @@
+HTTP_Server: "On"
+wrongField: "example"

cshared.go

@@ -27,9 +27,11 @@ import (
 	"github.com/calyptia/plugin/output"
 )
 
-var unregister func()
-var cmt *cmetrics.Context
-var logger Logger
+var (
+	unregister func()
+	cmt        *cmetrics.Context
+	logger     Logger
+)
 
 // FLBPluginRegister registers a plugin in the context of the fluent-bit runtime, a name and description
 // can be provided.