Merge pull request #175 from kubescape/sbomset

compress opens only if path isn't in sbom

main.go

@@ -66,7 +66,7 @@ func main() {
 	pool := file.NewPool(filepath.Join(file.DefaultStorageRoot, "metadata.sq3"), 0) // If less than 1, a reasonable default is used.
 
 	stopCh := genericapiserver.SetupSignalHandler()
-	options := server.NewWardleServerOptions(os.Stdout, os.Stderr, osFs, pool)
+	options := server.NewWardleServerOptions(os.Stdout, os.Stderr, osFs, pool, clusterData.Namespace)
 	cmd := server.NewCommandStartWardleServer(options, stopCh)
 
 	// cleanup task

pkg/apiserver/apiserver.go

@@ -84,8 +84,9 @@ func init() {
 
 // ExtraConfig holds custom apiserver config
 type ExtraConfig struct {
-	OsFs afero.Fs
-	Pool *sqlitemigration.Pool
+	Namespace string
+	OsFs      afero.Fs
+	Pool      *sqlitemigration.Pool
 }
 
 // Config defines the config for the apiserver
@@ -146,7 +147,7 @@ func (c completedConfig) New() (*WardleServer, error) {
 	var (
 		storageImpl = file.NewStorageImpl(c.ExtraConfig.OsFs, file.DefaultStorageRoot, c.ExtraConfig.Pool, Scheme)
 
-		applicationProfileStorageImpl  = file.NewStorageImplWithCollector(c.ExtraConfig.OsFs, file.DefaultStorageRoot, c.ExtraConfig.Pool, Scheme, file.NewApplicationProfileProcessor())
+		applicationProfileStorageImpl  = file.NewStorageImplWithCollector(c.ExtraConfig.OsFs, file.DefaultStorageRoot, c.ExtraConfig.Pool, Scheme, file.NewApplicationProfileProcessor(c.ExtraConfig.Namespace))
 		networkNeighborhoodStorageImpl = file.NewStorageImplWithCollector(c.ExtraConfig.OsFs, file.DefaultStorageRoot, c.ExtraConfig.Pool, Scheme, file.NewNetworkNeighborhoodProcessor())
 		configScanStorageImpl          = file.NewConfigurationScanSummaryStorage(storageImpl)
 		vulnerabilitySummaryStorage    = file.NewVulnerabilitySummaryStorage(storageImpl)

pkg/cmd/server/start.go

@@ -61,12 +61,13 @@ type WardleServerOptions struct {
 
 	AlternateDNS []string
 
-	OsFs afero.Fs
-	Pool *sqlitemigration.Pool
+	OsFs      afero.Fs
+	Pool      *sqlitemigration.Pool
+	Namespace string
 }
 
 // NewWardleServerOptions returns a new WardleServerOptions
-func NewWardleServerOptions(out, errOut io.Writer, osFs afero.Fs, pool *sqlitemigration.Pool) *WardleServerOptions {
+func NewWardleServerOptions(out, errOut io.Writer, osFs afero.Fs, pool *sqlitemigration.Pool, namespace string) *WardleServerOptions {
 	o := &WardleServerOptions{
 		RecommendedOptions: genericoptions.NewRecommendedOptions(
 			defaultEtcdPathPrefix,
@@ -76,8 +77,9 @@ func NewWardleServerOptions(out, errOut io.Writer, osFs afero.Fs, pool *sqlitemi
 		StdOut: out,
 		StdErr: errOut,
 
-		OsFs: osFs,
-		Pool: pool,
+		OsFs:      osFs,
+		Pool:      pool,
+		Namespace: namespace,
 	}
 	o.RecommendedOptions.Etcd = nil
 
@@ -218,8 +220,9 @@ func (o *WardleServerOptions) Config() (*apiserver.Config, error) {
 	config := &apiserver.Config{
 		GenericConfig: serverConfig,
 		ExtraConfig: apiserver.ExtraConfig{
-			OsFs: o.OsFs,
-			Pool: o.Pool,
+			OsFs:      o.OsFs,
+			Pool:      o.Pool,
+			Namespace: o.Namespace,
 		},
 	}
 	return config, nil

pkg/registry/file/applicationprofile_processor.go

@@ -1,16 +1,20 @@
 package file
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"strconv"
 
+	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/kubescape/go-logger"
 	loggerhelpers "github.com/kubescape/go-logger/helpers"
 	"github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers"
+	"github.com/kubescape/k8s-interface/names"
 	"github.com/kubescape/storage/pkg/apis/softwarecomposition"
 	"github.com/kubescape/storage/pkg/registry/file/dynamicpathdetector"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/storage"
 )
 
 const (
@@ -20,23 +24,26 @@ const (
 )
 
 type ApplicationProfileProcessor struct {
+	defaultNamespace          string
 	maxApplicationProfileSize int
+	storageImpl               *StorageImpl
 }
 
-func NewApplicationProfileProcessor() *ApplicationProfileProcessor {
+func NewApplicationProfileProcessor(defaultNamespace string) *ApplicationProfileProcessor {
 	maxApplicationProfileSize, err := strconv.Atoi(os.Getenv("MAX_APPLICATION_PROFILE_SIZE"))
 	if err != nil {
 		maxApplicationProfileSize = DefaultMaxApplicationProfileSize
 	}
 	logger.L().Debug("maxApplicationProfileSize", loggerhelpers.Int("size", maxApplicationProfileSize))
 	return &ApplicationProfileProcessor{
+		defaultNamespace:          defaultNamespace,
 		maxApplicationProfileSize: maxApplicationProfileSize,
 	}
 }
 
 var _ Processor = (*ApplicationProfileProcessor)(nil)
 
-func (a ApplicationProfileProcessor) PreSave(object runtime.Object) error {
+func (a *ApplicationProfileProcessor) PreSave(object runtime.Object) error {
 	profile, ok := object.(*softwarecomposition.ApplicationProfile)
 	if !ok {
 		return fmt.Errorf("given object is not an ApplicationProfile")
@@ -48,7 +55,25 @@ func (a ApplicationProfileProcessor) PreSave(object runtime.Object) error {
 	// Define a function to process a slice of containers
 	processContainers := func(containers []softwarecomposition.ApplicationProfileContainer) []softwarecomposition.ApplicationProfileContainer {
 		for i, container := range containers {
-			containers[i] = deflateApplicationProfileContainer(container)
+			var sbomSet mapset.Set[string]
+			// get files from corresponding sbom
+			sbomName, err := names.ImageInfoToSlug(container.ImageTag, container.ImageID)
+			if err == nil {
+				sbom := softwarecomposition.SBOMSyft{}
+				key := fmt.Sprintf("/spdx.softwarecomposition.kubescape.io/sbomsyft/%s/%s", a.defaultNamespace, sbomName)
+				if err := a.storageImpl.Get(context.Background(), key, storage.GetOptions{}, &sbom); err == nil {
+					// fill sbomSet
+					sbomSet = mapset.NewSet[string]()
+					for _, f := range sbom.Spec.Syft.Files {
+						sbomSet.Add(f.Location.RealPath)
+					}
+				} else {
+					logger.L().Debug("failed to get sbom", loggerhelpers.Error(err), loggerhelpers.String("key", key))
+				}
+			} else {
+				logger.L().Debug("failed to get sbom name", loggerhelpers.Error(err), loggerhelpers.String("imageTag", container.ImageTag), loggerhelpers.String("imageID", container.ImageID))
+			}
+			containers[i] = deflateApplicationProfileContainer(container, sbomSet)
 			size += len(containers[i].Execs)
 			size += len(containers[i].Opens)
 		}
@@ -75,10 +100,14 @@ func (a ApplicationProfileProcessor) PreSave(object runtime.Object) error {
 	return nil
 }
 
-func deflateApplicationProfileContainer(container softwarecomposition.ApplicationProfileContainer) softwarecomposition.ApplicationProfileContainer {
-	opens, err := dynamicpathdetector.AnalyzeOpens(container.Opens, dynamicpathdetector.NewPathAnalyzer(OpenDynamicThreshold))
+func (a *ApplicationProfileProcessor) SetStorage(storageImpl *StorageImpl) {
+	a.storageImpl = storageImpl
+}
+
+func deflateApplicationProfileContainer(container softwarecomposition.ApplicationProfileContainer, sbomSet mapset.Set[string]) softwarecomposition.ApplicationProfileContainer {
+	opens, err := dynamicpathdetector.AnalyzeOpens(container.Opens, dynamicpathdetector.NewPathAnalyzer(OpenDynamicThreshold), sbomSet)
 	if err != nil {
-		logger.L().Warning("failed to analyze opens", loggerhelpers.Error(err))
+		logger.L().Debug("failed to analyze opens", loggerhelpers.Error(err))
 		opens = DeflateStringer(container.Opens)
 	}
 

pkg/registry/file/applicationprofile_processor_test.go

@@ -145,7 +145,7 @@ func TestApplicationProfileProcessor_PreSave(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Setenv("MAX_APPLICATION_PROFILE_SIZE", strconv.Itoa(tt.maxApplicationProfileSize))
-			a := NewApplicationProfileProcessor()
+			a := NewApplicationProfileProcessor("kubescape")
 			tt.wantErr(t, a.PreSave(tt.object), fmt.Sprintf("PreSave(%v)", tt.object))
 			slices.Sort(tt.object.(*softwarecomposition.ApplicationProfile).Spec.Architectures)
 			assert.Equal(t, tt.want, tt.object)

pkg/registry/file/dynamicpathdetector/analyze_opens.go

@@ -1,6 +1,7 @@
 package dynamicpathdetector
 
 import (
+	"errors"
 	"maps"
 	"slices"
 	"strings"
@@ -9,17 +10,27 @@ import (
 	types "github.com/kubescape/storage/pkg/apis/softwarecomposition"
 )
 
-func AnalyzeOpens(opens []types.OpenCalls, analyzer *PathAnalyzer) ([]types.OpenCalls, error) {
+func AnalyzeOpens(opens []types.OpenCalls, analyzer *PathAnalyzer, sbomSet mapset.Set[string]) ([]types.OpenCalls, error) {
 	if opens == nil {
 		return nil, nil
 	}
 
+	if sbomSet == nil {
+		return nil, errors.New("sbomSet is nil")
+	}
+
 	dynamicOpens := make(map[string]types.OpenCalls)
 	for _, open := range opens {
 		_, _ = AnalyzeOpen(open.Path, analyzer)
 	}
 
 	for i := range opens {
+		// sbomSet files have to be always present in the dynamicOpens
+		if sbomSet.ContainsOne(opens[i].Path) {
+			dynamicOpens[opens[i].Path] = opens[i]
+			continue
+		}
+
 		result, err := AnalyzeOpen(opens[i].Path, analyzer)
 		if err != nil {
 			continue

pkg/registry/file/dynamicpathdetector/tests/analyze_opens_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	mapset "github.com/deckarep/golang-set/v2"
 	types "github.com/kubescape/storage/pkg/apis/softwarecomposition"
 	"github.com/kubescape/storage/pkg/registry/file/dynamicpathdetector"
 	"github.com/stretchr/testify/assert"
@@ -26,7 +27,34 @@ func TestAnalyzeOpensWithThreshold(t *testing.T) {
 		},
 	}
 
-	result, err := dynamicpathdetector.AnalyzeOpens(input, analyzer)
+	result, err := dynamicpathdetector.AnalyzeOpens(input, analyzer, mapset.NewSet[string]())
+	assert.NoError(t, err)
+	assert.Equal(t, expected, result)
+}
+
+func TestAnalyzeOpensWithThresholdAndExclusion(t *testing.T) {
+	analyzer := dynamicpathdetector.NewPathAnalyzer(100)
+
+	var input []types.OpenCalls
+	for i := 0; i < 101; i++ {
+		input = append(input, types.OpenCalls{
+			Path:  fmt.Sprintf("/home/user%d/file.txt", i),
+			Flags: []string{"READ"},
+		})
+	}
+
+	expected := []types.OpenCalls{
+		{
+			Path:  "/home/user42/file.txt",
+			Flags: []string{"READ"},
+		},
+		{
+			Path:  "/home/\u22ef/file.txt",
+			Flags: []string{"READ"},
+		},
+	}
+
+	result, err := dynamicpathdetector.AnalyzeOpens(input, analyzer, mapset.NewSet[string]("/home/user42/file.txt"))
 	assert.NoError(t, err)
 	assert.Equal(t, expected, result)
 }
@@ -98,7 +126,7 @@ func TestAnalyzeOpensWithFlagMergingAndThreshold(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			analyzer := dynamicpathdetector.NewPathAnalyzer(3)
-			result, err := dynamicpathdetector.AnalyzeOpens(tt.input, analyzer)
+			result, err := dynamicpathdetector.AnalyzeOpens(tt.input, analyzer, mapset.NewSet[string]())
 			assert.NoError(t, err)
 
 			assert.Equal(t, tt.expected, result)

pkg/registry/file/dynamicpathdetector/tests/benchmark_test.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"testing"
 
+	mapset "github.com/deckarep/golang-set/v2"
 	types "github.com/kubescape/storage/pkg/apis/softwarecomposition"
 	"github.com/kubescape/storage/pkg/registry/file"
 	"github.com/kubescape/storage/pkg/registry/file/dynamicpathdetector"
@@ -57,7 +58,7 @@ func BenchmarkAnalyzeOpensVsDeflateStringer(b *testing.B) {
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			_ = file.DeflateStringer(paths)
-			_, _ = dynamicpathdetector.AnalyzeOpens(paths, analyzer)
+			_, _ = dynamicpathdetector.AnalyzeOpens(paths, analyzer, mapset.NewSet[string]())
 		}
 		b.ReportAllocs()
 	})

pkg/registry/file/networkneighborhood_processor.go

@@ -71,6 +71,8 @@ func (a NetworkNeighborhoodProcessor) PreSave(object runtime.Object) error {
 	return nil
 }
 
+func (a NetworkNeighborhoodProcessor) SetStorage(_ *StorageImpl) {}
+
 func deflateNetworkNeighborhoodContainer(container softwarecomposition.NetworkNeighborhoodContainer) softwarecomposition.NetworkNeighborhoodContainer {
 	return softwarecomposition.NetworkNeighborhoodContainer{
 		Name:    container.Name,

pkg/registry/file/processor.go

@@ -8,6 +8,7 @@ import (
 
 type Processor interface {
 	PreSave(object runtime.Object) error
+	SetStorage(storageImpl *StorageImpl)
 }
 
 type DefaultProcessor struct {
@@ -19,6 +20,8 @@ func (d DefaultProcessor) PreSave(_ runtime.Object) error {
 	return nil
 }
 
+func (d DefaultProcessor) SetStorage(_ *StorageImpl) {}
+
 type Stringer interface {
 	String() string
 }

pkg/registry/file/storage.go

@@ -83,7 +83,7 @@ func NewStorageImpl(appFs afero.Fs, root string, pool *sqlitemigration.Pool, sch
 }
 
 func NewStorageImplWithCollector(appFs afero.Fs, root string, conn *sqlitemigration.Pool, scheme *runtime.Scheme, processor Processor) StorageQuerier {
-	return &StorageImpl{
+	storageImpl := &StorageImpl{
 		appFs:           appFs,
 		pool:            conn,
 		locks:           utils.NewMapMutex[string](),
@@ -93,6 +93,8 @@ func NewStorageImplWithCollector(appFs afero.Fs, root string, conn *sqlitemigrat
 		versioner:       storage.APIObjectVersioner{},
 		watchDispatcher: newWatchDispatcher(),
 	}
+	processor.SetStorage(storageImpl)
+	return storageImpl
 }
 
 // Versioner Returns Versioner associated with this interface.
@@ -554,6 +556,15 @@ func (s *StorageImpl) GuaranteedUpdate(
 		return err
 	}
 
+	// check object size
+	annotations := origState.obj.(metav1.Object).GetAnnotations()
+	if annotations != nil && annotations[helpersv1.StatusMetadataKey] == helpersv1.TooLarge {
+		logger.L().Ctx(ctx).Debug("GuaranteedUpdate - already too large object, skipping update", helpers.String("key", key))
+		// no change, return the original object
+		v.Set(reflect.ValueOf(origState.obj).Elem())
+		return nil
+	}
+
 	for {
 		// run preconditions
 		if err := preconditions.Check(key, origState.obj); err != nil {
@@ -621,7 +632,7 @@ func (s *StorageImpl) GuaranteedUpdate(
 				annotations := metadata.GetAnnotations()
 				annotations[helpersv1.StatusMetadataKey] = helpersv1.TooLarge
 				metadata.SetAnnotations(annotations)
-				logger.L().Ctx(ctx).Warning("GuaranteedUpdate - too large object, skipping update", helpers.String("key", key))
+				logger.L().Ctx(ctx).Debug("GuaranteedUpdate - too large object, skipping update", helpers.String("key", key))
 			} else {
 				return fmt.Errorf("processor.PreSave: %w", err)
 			}