Added tests for endpoints

pkg/registry/file/applicationprofile_processor.go

@@ -49,7 +49,7 @@ func (a ApplicationProfileProcessor) PreSave(object runtime.Object) error {
 }
 
 func deflateApplicationProfileContainer(container softwarecomposition.ApplicationProfileContainer) softwarecomposition.ApplicationProfileContainer {
-	endpoints, err := dynamicpathdetector.AnalyzeEndpoints(&container.Endpoints, dynamicpathdetector.NewPathAnalyzer())
+	endpoints, err := dynamicpathdetector.AnalyzeEndpoints(&container.Endpoints, dynamicpathdetector.NewPathAnalyzer(100))
 	if err != nil {
 		logger.L().Warning("failed to analyze endpoints", loggerhelpers.Error(err))
 		endpoints = container.Endpoints

pkg/registry/file/dynamicpathdetector/analyze_endpoints.go

@@ -11,8 +11,7 @@ import (
 )
 
 func AnalyzeEndpoints(endpoints *[]types.HTTPEndpoint, analyzer *PathAnalyzer) ([]types.HTTPEndpoint, error) {
-	var newEndpoints []types.HTTPEndpoint
-	MergeDuplicateEndpoints(endpoints)
+	var newEndpoints []*types.HTTPEndpoint
 	for _, endpoint := range *endpoints {
 		AnalyzeURL(endpoint.Endpoint, analyzer)
 	}
@@ -22,14 +21,19 @@ func AnalyzeEndpoints(endpoints *[]types.HTTPEndpoint, analyzer *PathAnalyzer) (
 		if processedEndpoint == nil && err == nil || err != nil {
 			continue
 		} else {
-			newEndpoints = append(newEndpoints, *processedEndpoint)
+			newEndpoints = append(newEndpoints, processedEndpoint)
 		}
 	}
 
-	return newEndpoints, nil
+	newEndpoints, err := MergeDuplicateEndpoints(newEndpoints)
+	if err != nil {
+		return nil, err
+	}
+
+	return convertPointerToValueSlice(newEndpoints), nil
 }
 
-func ProcessEndpoint(endpoint *types.HTTPEndpoint, analyzer *PathAnalyzer, newEndpoints []types.HTTPEndpoint) (*types.HTTPEndpoint, error) {
+func ProcessEndpoint(endpoint *types.HTTPEndpoint, analyzer *PathAnalyzer, newEndpoints []*types.HTTPEndpoint) (*types.HTTPEndpoint, error) {
 	url, err := AnalyzeURL(endpoint.Endpoint, analyzer)
 	if err != nil {
 		return nil, err
@@ -41,7 +45,7 @@ func ProcessEndpoint(endpoint *types.HTTPEndpoint, analyzer *PathAnalyzer, newEn
 		for i, e := range newEndpoints {
 			if e.Endpoint == url {
 				newEndpoints[i].Methods = mergeMethods(e.Methods, endpoint.Methods)
-				mergeHeaders(&e, endpoint)
+				mergeHeaders(e, endpoint)
 				return nil, nil
 			}
 		}
@@ -65,6 +69,10 @@ func AnalyzeURL(urlString string, analyzer *PathAnalyzer) (string, error) {
 		urlString = "http://" + urlString
 	}
 
+	if err := isValidURL(urlString); err != nil {
+		return "", err
+	}
+
 	parsedURL, err := url.Parse(urlString)
 	if err != nil {
 		return "", err
@@ -79,24 +87,21 @@ func AnalyzeURL(urlString string, analyzer *PathAnalyzer) (string, error) {
 	return hostname + path, nil
 }
 
-func MergeDuplicateEndpoints(endpoints *[]types.HTTPEndpoint) {
+func MergeDuplicateEndpoints(endpoints []*types.HTTPEndpoint) ([]*types.HTTPEndpoint, error) {
 	seen := make(map[string]*types.HTTPEndpoint)
-	newEndpoints := make([]types.HTTPEndpoint, 0)
-
-	for i := range *endpoints {
-		endpoint := &(*endpoints)[i]
+	var newEndpoints []*types.HTTPEndpoint
+	for _, endpoint := range endpoints {
 		key := getEndpointKey(endpoint)
 
 		if existing, found := seen[key]; found {
 			existing.Methods = mergeMethods(existing.Methods, endpoint.Methods)
 			mergeHeaders(existing, endpoint)
 		} else {
 			seen[key] = endpoint
-			newEndpoints = append(newEndpoints, *endpoint)
+			newEndpoints = append(newEndpoints, endpoint)
 		}
 	}
-
-	*endpoints = newEndpoints
+	return newEndpoints, nil
 }
 
 func getEndpointKey(endpoint *types.HTTPEndpoint) string {
@@ -143,5 +148,21 @@ func mergeMethods(existing, new []string) []string {
 			methodSet[m] = true
 		}
 	}
+
 	return existing
 }
+
+func convertPointerToValueSlice(m []*types.HTTPEndpoint) []types.HTTPEndpoint {
+	result := make([]types.HTTPEndpoint, 0, len(m))
+	for _, v := range m {
+		if v != nil {
+			result = append(result, *v)
+		}
+	}
+	return result
+}
+
+func isValidURL(rawURL string) error {
+	_, err := url.ParseRequestURI(rawURL)
+	return err
+}

pkg/registry/file/dynamicpathdetector/analyzer.go

@@ -5,10 +5,10 @@ import (
 	"strings"
 )
 
-func NewPathAnalyzer() *PathAnalyzer {
+func NewPathAnalyzer(threshold int) *PathAnalyzer {
 	return &PathAnalyzer{
 		RootNodes: make(map[string]*SegmentNode),
-		threshold: 100,
+		threshold: threshold,
 	}
 }
 func (ua *PathAnalyzer) AnalyzePath(path, identifier string) (string, error) {

pkg/registry/file/dynamicpathdetector/tests/analyze_endpoints_test.go

@@ -0,0 +1,237 @@
+package dynamicpathdetectortests
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"testing"
+
+	types "github.com/kubescape/storage/pkg/apis/softwarecomposition"
+	"github.com/kubescape/storage/pkg/registry/file/dynamicpathdetector"
+)
+
+func TestAnalyzeEndpoints(t *testing.T) {
+	analyzer := dynamicpathdetector.NewPathAnalyzer(100)
+
+	tests := []struct {
+		name     string
+		input    []types.HTTPEndpoint
+		expected []types.HTTPEndpoint
+	}{
+		{
+			name: "Basic test with single endpoint",
+			input: []types.HTTPEndpoint{
+				{
+					Endpoint: "api.example.com/users/123",
+					Methods:  []string{"GET"},
+				},
+			},
+			expected: []types.HTTPEndpoint{
+				{
+					Endpoint: "api.example.com/users/123",
+					Methods:  []string{"GET"},
+				},
+			},
+		},
+		{
+			name: "Test with multiple endpoints",
+			input: []types.HTTPEndpoint{
+				{
+					Endpoint: "api.example.com/users/<dynamic>",
+					Methods:  []string{"GET"},
+				},
+				{
+					Endpoint: "api.example.com/users/123",
+					Methods:  []string{"POST"},
+				},
+			},
+			expected: []types.HTTPEndpoint{
+				{
+					Endpoint: "api.example.com/users/<dynamic>",
+					Methods:  []string{"GET", "POST"},
+				},
+			},
+		},
+		{
+			name: "Test with dynamic segments",
+			input: []types.HTTPEndpoint{
+				{
+					Endpoint: "api.example.com/users/123/posts/<dynamic>",
+					Methods:  []string{"GET"},
+				},
+				{
+					Endpoint: "api.example.com/users/<dynamic>/posts/101",
+					Methods:  []string{"POST"},
+				},
+			},
+			expected: []types.HTTPEndpoint{
+				{
+					Endpoint: "api.example.com/users/<dynamic>/posts/<dynamic>",
+					Methods:  []string{"GET", "POST"},
+				},
+			},
+		},
+		{
+			name: "Test with different domains",
+			input: []types.HTTPEndpoint{
+				{
+					Endpoint: "api1.example.com/users/123",
+					Methods:  []string{"GET"},
+				},
+				{
+					Endpoint: "api2.example.com/users/456",
+					Methods:  []string{"POST"},
+				},
+				{
+					Endpoint: "api2.example.com/x/x",
+					Methods:  []string{"GET"},
+				},
+				{
+					Endpoint: "api2.example.com/x/x",
+					Methods:  []string{"POST"},
+				},
+			},
+			expected: []types.HTTPEndpoint{
+				{
+					Endpoint: "api1.example.com/users/123",
+					Methods:  []string{"GET"},
+				},
+				{
+					Endpoint: "api2.example.com/users/456",
+					Methods:  []string{"POST"},
+				},
+				{
+					Endpoint: "api2.example.com/x/x",
+					Methods:  []string{"GET", "POST"},
+				},
+			},
+		},
+		{
+			name: "Test with dynamic segments and different headers",
+			input: []types.HTTPEndpoint{
+				{
+					Endpoint: "api.example.com/x/123/posts/<dynamic>",
+					Methods:  []string{"GET"},
+					Headers:  json.RawMessage(`{"Content-Type": ["application/json"], "X-API-Key": ["key1"]}`),
+				},
+				{
+					Endpoint: "api.example.com/x/<dynamic>/posts/101",
+					Methods:  []string{"POST"},
+					Headers:  json.RawMessage(`{"Content-Type": ["application/xml"], "Authorization": ["Bearer token"]}`),
+				},
+			},
+			expected: []types.HTTPEndpoint{
+				{
+					Endpoint: "api.example.com/x/<dynamic>/posts/<dynamic>",
+					Methods:  []string{"GET", "POST"},
+					Headers:  json.RawMessage([]byte{123, 34, 65, 117, 116, 104, 111, 114, 105, 122, 97, 116, 105, 111, 110, 34, 58, 91, 34, 66, 101, 97, 114, 101, 114, 32, 116, 111, 107, 101, 110, 34, 93, 44, 34, 67, 111, 110, 116, 101, 110, 116, 45, 84, 121, 112, 101, 34, 58, 91, 34, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 106, 115, 111, 110, 34, 44, 34, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 120, 109, 108, 34, 93, 44, 34, 88, 45, 65, 80, 73, 45, 75, 101, 121, 34, 58, 91, 34, 107, 101, 121, 49, 34, 93, 125}),
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := dynamicpathdetector.AnalyzeEndpoints(&tt.input, analyzer)
+			if err != nil {
+				t.Errorf("AnalyzeEndpoints() error = %v", err)
+				return
+			}
+			if !reflect.DeepEqual(result, tt.expected) {
+				t.Errorf("AnalyzeEndpoints() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestAnalyzeEndpointsWithThreshold(t *testing.T) {
+	analyzer := dynamicpathdetector.NewPathAnalyzer(100)
+
+	var input []types.HTTPEndpoint
+	for i := 0; i < 101; i++ {
+		input = append(input, types.HTTPEndpoint{
+			Endpoint: fmt.Sprintf("api.example.com/users/%d", i),
+			Methods:  []string{"GET"},
+		})
+	}
+
+	expected := []types.HTTPEndpoint{
+		{
+			Endpoint: "api.example.com/users/<dynamic>",
+			Methods:  []string{"GET"},
+		},
+	}
+
+	result, err := dynamicpathdetector.AnalyzeEndpoints(&input, analyzer)
+	if err != nil {
+		t.Errorf("AnalyzeEndpoints() error = %v", err)
+		return
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("AnalyzeEndpoints() = %v, want %v", result, expected)
+	}
+}
+
+func TestAnalyzeEndpointsWithExactThreshold(t *testing.T) {
+	analyzer := dynamicpathdetector.NewPathAnalyzer(100)
+
+	var input []types.HTTPEndpoint
+	for i := 0; i < 100; i++ {
+		input = append(input, types.HTTPEndpoint{
+			Endpoint: fmt.Sprintf("api.example.com/users/%d", i),
+			Methods:  []string{"GET"},
+		})
+	}
+
+	result, err := dynamicpathdetector.AnalyzeEndpoints(&input, analyzer)
+	if err != nil {
+		t.Errorf("AnalyzeEndpoints() error = %v", err)
+		return
+	}
+
+	// Check that all 100 endpoints are still individual
+	if len(result) != 100 {
+		t.Errorf("Expected 100 individual endpoints, got %d", len(result))
+	}
+
+	// Now add one more endpoint to trigger the dynamic behavior
+	input = append(input, types.HTTPEndpoint{
+		Endpoint: "api.example.com/users/100",
+		Methods:  []string{"GET"},
+	})
+
+	result, err = dynamicpathdetector.AnalyzeEndpoints(&input, analyzer)
+	if err != nil {
+		t.Errorf("AnalyzeEndpoints() error = %v", err)
+		return
+	}
+
+	// Check that all endpoints are now merged into one dynamic endpoint
+	expected := []types.HTTPEndpoint{
+		{
+			Endpoint: "api.example.com/users/<dynamic>",
+			Methods:  []string{"GET"},
+		},
+	}
+
+	if !reflect.DeepEqual(result, expected) {
+		t.Errorf("AnalyzeEndpoints() = %v, want %v", result, expected)
+	}
+}
+
+func TestAnalyzeEndpointsWithInvalidURL(t *testing.T) {
+	analyzer := dynamicpathdetector.NewPathAnalyzer(100)
+
+	input := []types.HTTPEndpoint{
+		{
+			Endpoint: ":::invalid-u323@!#rl:::",
+			Methods:  []string{"GET"},
+		},
+	}
+
+	result, _ := dynamicpathdetector.AnalyzeEndpoints(&input, analyzer)
+
+	if len(result) != 0 {
+		t.Errorf("Expected empty result, got %v", result)
+	}
+}