Merge branch 'main' into edouard/fix-devenv-stuff

Makefile

@@ -10,10 +10,11 @@ COMMIT := $(shell git rev-parse --short HEAD)
 DATE := $(shell git log -1 --format=%cd --date=format:"%Y%m%d")
 
 BUILD_VERSION ?= $(shell git describe --match 'v[0-9]*' --dirty --always --tags)
+BUILD_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 BUILD_ARCH := $(shell go env GOARCH)
 BUILD_OS := $(shell go env GOOS)
 
-BUILD_FLAGS := -ldflags="-X github.com/DataDog/KubeHound/pkg/config.BuildVersion=$(BUILD_VERSION) -X github.com/DataDog/KubeHound/pkg/config.BuildArch=$(BUILD_ARCH) -X github.com/DataDog/KubeHound/pkg/config.BuildOs=$(BUILD_OS) -s -w"
+BUILD_FLAGS := -ldflags="-X github.com/DataDog/KubeHound/pkg/config.BuildVersion=$(BUILD_VERSION) -X github.com/DataDog/KubeHound/pkg/config.BuildBranch=$(BUILD_BRANCH) -X github.com/DataDog/KubeHound/pkg/config.BuildArch=$(BUILD_ARCH) -X github.com/DataDog/KubeHound/pkg/config.BuildOs=$(BUILD_OS) -s -w"
 
 # Need to save the MAKEFILE_LIST variable before the including the env var files
 HELP_MAKEFILE_LIST := $(MAKEFILE_LIST)

cmd/kubehound/backend.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/DataDog/KubeHound/pkg/backend"
 	docker "github.com/DataDog/KubeHound/pkg/backend"
 	"github.com/spf13/cobra"
 )
@@ -9,6 +10,9 @@ var (
 	Backend     *docker.Backend
 	hard        bool
 	composePath []string
+
+	uiProfile = backend.DefaultUIProfile
+	uiInvana  bool
 )
 
 var (
@@ -17,7 +21,11 @@ var (
 		Short: "Handle the kubehound stack",
 		Long:  `Handle the kubehound stack - docker compose based stack for kubehound services (mongodb, graphdb and UI)`,
 		PersistentPreRunE: func(cobraCmd *cobra.Command, args []string) error {
-			return docker.NewBackend(cobraCmd.Context(), composePath)
+			if uiInvana {
+				uiProfile = append(uiProfile, "invana")
+			}
+
+			return docker.NewBackend(cobraCmd.Context(), composePath, uiProfile)
 		},
 	}
 
@@ -78,5 +86,6 @@ func init() {
 
 	backendCmd.AddCommand(backendDownCmd)
 	backendCmd.PersistentFlags().StringSliceVarP(&composePath, "file", "f", composePath, "Compose configuration files")
+	backendCmd.PersistentFlags().BoolVar(&uiInvana, "invana", false, "Activate Invana front end as KubeHound UI alternative")
 	rootCmd.AddCommand(backendCmd)
 }

cmd/kubehound/dev.go

@@ -4,22 +4,24 @@ import (
 	"context"
 	"os"
 
+	"github.com/DataDog/KubeHound/pkg/backend"
 	docker "github.com/DataDog/KubeHound/pkg/backend"
 	"github.com/spf13/cobra"
 )
 
 var (
 	DefaultComposeTestingPath = []string{"./deployments/kubehound/docker-compose.yaml", "./deployments/kubehound/docker-compose.testing.yaml"}
-	DefaultComposeDevPath     = []string{"./deployments/kubehound/docker-compose.yaml", "./deployments/kubehound/docker-compose.dev.yaml"}
-	DefaultComposeDevPathUI   = "./deployments/kubehound/docker-compose.ui.yaml"
-	DefaultComposeDevPathGRPC = "./deployments/kubehound/docker-compose.ingestor.yaml"
+	DefaultComposeDevPath     = []string{"./deployments/kubehound/docker-compose.yaml", "./deployments/kubehound/docker-compose.dev.graph.yaml", "./deployments/kubehound/docker-compose.dev.mongo.yaml"}
+	DefaultComposeDevPathUI   = "./deployments/kubehound/docker-compose.dev.ui.yaml"
+	DefaultComposeDevPathGRPC = "./deployments/kubehound/docker-compose.dev.ingestor.yaml"
 	DefaultDatadogComposePath = "./deployments/kubehound/docker-compose.datadog.yaml"
 )
 
 var (
 	uiTesting   bool
 	grpcTesting bool
 	downTesting bool
+	profiles    []string
 )
 
 var (
@@ -28,9 +30,6 @@ var (
 		Hidden: true,
 		Short:  "[devOnly] Spawn the kubehound testing stack",
 		Long:   `[devOnly] Spawn the kubehound dev stack for the system-tests (build from dockerfile)`,
-		PersistentPreRunE: func(cobraCmd *cobra.Command, args []string) error {
-			return docker.NewBackend(cobraCmd.Context(), composePath)
-		},
 		RunE: func(cobraCmd *cobra.Command, args []string) error {
 			if uiTesting {
 				DefaultComposeDevPath = append(DefaultComposeDevPath, DefaultComposeDevPathUI)
@@ -60,7 +59,11 @@ var (
 )
 
 func runEnv(ctx context.Context, composePaths []string) error {
-	err := docker.NewBackend(ctx, composePaths)
+	if uiTesting {
+		profiles = append(profiles, backend.DevUIProfile)
+	}
+
+	err := docker.NewBackend(ctx, composePaths, profiles)
 	if err != nil {
 		return err
 	}

cmd/kubehound/ingest.go

@@ -42,11 +42,17 @@ var (
 	remoteIngestCmd = &cobra.Command{
 		Use:   "remote",
 		Short: "Ingest data remotely on a KHaaS instance",
-		Long:  `Run an ingestion on KHaaS from a bucket to build the attack path`,
+		Long:  `Run an ingestion on KHaaS from a bucket to build the attack path, by default it will rehydrate the latest snapshot previously dumped on a KHaaS instance from all clusters`,
 		PreRunE: func(cobraCmd *cobra.Command, args []string) error {
 			viper.BindPFlag(config.IngestorAPIEndpoint, cobraCmd.Flags().Lookup("khaas-server")) //nolint: errcheck
+			cobraCmd.MarkFlagRequired("khaas-server")                                            //nolint: errcheck
 			viper.BindPFlag(config.IngestorAPIInsecure, cobraCmd.Flags().Lookup("insecure"))     //nolint: errcheck
 
+			if !isIngestRemoteDefault() {
+				cobraCmd.MarkFlagRequired("run_id")  //nolint: errcheck
+				cobraCmd.MarkFlagRequired("cluster") //nolint: errcheck
+			}
+
 			return cmd.InitializeKubehoundConfig(cobraCmd.Context(), "", false, true)
 		},
 		RunE: func(cobraCmd *cobra.Command, args []string) error {
@@ -56,11 +62,22 @@ var (
 				return fmt.Errorf("get config: %w", err)
 			}
 
+			if isIngestRemoteDefault() {
+				return core.CoreClientGRPCRehydrateLatest(khCfg.Ingestor)
+			}
+
 			return core.CoreClientGRPCIngest(khCfg.Ingestor, khCfg.Ingestor.ClusterName, khCfg.Ingestor.RunID)
 		},
 	}
 )
 
+func isIngestRemoteDefault() bool {
+	runID := viper.GetString(config.IngestorRunID)
+	clusterName := viper.GetString(config.IngestorClusterName)
+
+	return runID == "" && clusterName == ""
+}
+
 func init() {
 
 	ingestCmd.AddCommand(localIngestCmd)

cmd/kubehound/root.go

@@ -27,7 +27,7 @@ var (
 			// auto spawning the backend stack
 			if !skipBackend {
 				// Forcing the embed docker config to be loaded
-				err := backend.NewBackend(cobraCmd.Context(), []string{""})
+				err := backend.NewBackend(cobraCmd.Context(), []string{""}, backend.DefaultUIProfile)
 				if err != nil {
 					return err
 				}

deployments/kubehound/docker-compose.datadog.yaml

@@ -31,4 +31,4 @@ services:
       - kubenet
 
 networks:
-  kubenet:
+  kubenet:

deployments/kubehound/docker-compose.dev.graph.yaml

@@ -0,0 +1,7 @@
+name: kubehound-dev
+services:
+  kubegraph:
+    build: ./kubegraph/
+    ports:
+      - "127.0.0.1:8182:8182"
+      - "127.0.0.1:8099:8099"

deployments/kubehound/docker-compose.ingestor.yaml → deployments/kubehound/docker-compose.dev.ingestor.yaml

@@ -1,3 +1,4 @@
+name: kubehound-dev
 services:
   grpc:
     build:
@@ -10,4 +11,4 @@ services:
       - kubenet
 
 networks:
-  kubenet:
+  kubenet:

deployments/kubehound/docker-compose.dev.mongo.yaml

@@ -0,0 +1,5 @@
+name: kubehound-dev
+services:
+  mongodb:
+    ports:
+      - "127.0.0.1:27017:27017"

deployments/kubehound/docker-compose.ui.yaml → deployments/kubehound/docker-compose.dev.ui.yaml

@@ -1,13 +1,7 @@
+name: kubehound-dev
 services:
-  notebook:
+  ui-jupyter:
     build: ./notebook/
     restart: unless-stopped
-    ports:
-      - "127.0.0.1:8888:8888"
-    networks:
-      - kubenet
     volumes:
       - ./notebook/shared:/root/notebooks/shared
-
-networks:
-  kubenet:

deployments/kubehound/docker-compose.release.yaml

@@ -0,0 +1,11 @@
+name: kubehound-release
+services:
+  mongodb:
+    ports:
+      - "127.0.0.1:27017:27017"
+
+  kubegraph:
+    image: ghcr.io/datadog/kubehound-ui:latest
+    ports:
+      - "127.0.0.1:8182:8182"
+      - "127.0.0.1:8099:8099"

deployments/kubehound/docker-compose.release.yaml.tpl

@@ -3,33 +3,12 @@ services:
   mongodb:
     ports:
       - "127.0.0.1:27017:27017"
-    volumes:
-      - mongodb_data:/data/db
 
   kubegraph:
     image: ghcr.io/datadog/kubehound-graph:{{ .VersionTag }}
     ports:
       - "127.0.0.1:8182:8182"
       - "127.0.0.1:8099:8099"
-    volumes:
-      - kubegraph_data:/var/lib/janusgraph
 
-  ui:
+  ui-jupyter:
     image: ghcr.io/datadog/kubehound-ui:{{ .VersionTag }}
-    restart: unless-stopped
-    ports:
-      - "127.0.0.1:8888:8888"
-    networks:
-      - kubenet
-    labels:
-      com.datadoghq.ad.logs: '[{"app": "kubeui", "service": "kubehound"}]'
-    volumes:
-      - kubeui_data:/root/notebooks/shared
-
-volumes:
-  mongodb_data:
-  kubegraph_data:
-  kubeui_data:
-
-networks:
-  kubenet:

deployments/kubehound/docker-compose.testing.yaml

@@ -3,17 +3,9 @@ services:
   mongodb:
     ports:
       - "127.0.0.1:27018:27017"
-    networks:
-      - kind
 
   kubegraph:
     build: ./kubegraph/
-    networks:
-      - kind
     ports:
       - "127.0.0.1:8183:8182"
       - "127.0.0.1:8090:8099"
-
-networks:
-  kind:
-    external: true

deployments/kubehound/docker-compose.yaml

@@ -14,6 +14,8 @@ services:
       interval: 10s
       timeout: 2s
       retries: 10
+    volumes:
+      - mongodb_data:/data/db
 
   kubegraph:
     restart: unless-stopped
@@ -26,6 +28,56 @@ services:
       retries: 3
     labels:
       com.datadoghq.ad.logs: '[{"app": "kubegraph", "service": "kubehound"}]'
+    volumes:
+      - kubegraph_data:/var/lib/janusgraph
+
+  ui-jupyter:
+    restart: unless-stopped
+    profiles:
+      - jupyter
+    ports:
+      - "127.0.0.1:8888:8888"
+      - "127.0.0.1:8889:8889"
+    networks:
+      - kubenet
+    labels:
+      com.datadoghq.ad.logs: '[{"app": "kubeui", "service": "kubehound"}]'
+    volumes:
+      - kubeui_data:/root/notebooks/shared
+    environment:
+      - NOTEBOOK_PASSWORD=admin
+      - GRAPH_NOTEBOOK_SSL=False
+
+  ui-invana-engine:
+    image: invanalabs/invana-engine:latest
+    profiles:
+      - invana
+    restart: unless-stopped
+    networks:
+      - kubenet
+    ports:
+      - 127.0.0.1:8200:8200
+    environment:
+      GREMLIN_SERVER_URL: ws://kubegraph:8182/gremlin
+    depends_on:
+      - kubegraph
+
+  ui-invana-studio:
+    image: invanalabs/invana-studio:latest
+    restart: unless-stopped
+    profiles:
+      - invana
+    networks:
+      - kubenet
+    ports:
+      - 127.0.0.1:8300:8300
+    depends_on:
+      - ui-invana-engine
 
 networks:
-  kubenet:
+  kubenet:
+
+volumes:
+  mongodb_data:
+  kubegraph_data:
+  kubeui_data: