Merge remote-tracking branch 'origin/refactor/yaml-merge-hell-for-spe…

…cs-file' into SimPeccaud/main
SimPeccaud · Jun 10, 2024 · 8895ad0 · 8895ad0
2 parents 52cca66 + 64d92d0
commit 8895ad0
Show file tree

Hide file tree

Showing 10 changed files with 119 additions and 116 deletions.
diff --git a/.develop/dev.Dockerfile b/.develop/dev.Dockerfile
@@ -1,8 +1,16 @@
 FROM --platform=linux/amd64 golang:1.22-bookworm
 
-RUN set -x && apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    ca-certificates wget jq procps curl vim inetutils-ping binutils && \
-    rm -rf /var/lib/apt/lists/*
+RUN apt-get update -y && \
+    apt-get install -yq --no-install-recommends \
+            ca-certificates \
+            wget \
+            jq \
+            procps \
+            curl \
+            vim \
+            inetutils-ping binutils && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/cache/apt/archive/*.deb
 
 WORKDIR /clabernetes
 

diff --git a/build/clabverter.Dockerfile b/build/clabverter.Dockerfile
@@ -4,16 +4,11 @@ ARG VERSION
 
 WORKDIR /clabernetes
 
-RUN mkdir build
-RUN mkdir work
-
-COPY cmd/clabverter/main.go main.go
+RUN mkdir build && \
+    mkdir work
 
 COPY . .
 
-COPY go.mod go.mod
-COPY go.sum go.sum
-
 RUN go mod download
 
 RUN CGO_ENABLED=0 \
@@ -23,7 +18,7 @@ RUN CGO_ENABLED=0 \
     -a \
     -o \
     build/clabverter \
-    main.go
+    cmd/clabverter/main.go
 
 FROM --platform=linux/amd64 gcr.io/distroless/static-debian12:nonroot
 

diff --git a/build/launcher.Dockerfile b/build/launcher.Dockerfile
@@ -6,13 +6,8 @@ WORKDIR /clabernetes
 
 RUN mkdir build
 
-COPY cmd/clabernetes/main.go main.go
-
 COPY . .
 
-COPY go.mod go.mod
-COPY go.sum go.sum
-
 RUN go mod download
 
 RUN CGO_ENABLED=0 \
@@ -24,28 +19,30 @@ RUN CGO_ENABLED=0 \
     -a \
     -o \
     build/manager \
-    main.go
+    cmd/clabernetes/main.go
 
 FROM --platform=linux/amd64 debian:bookworm-slim
 
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 ARG DOCKER_VERSION="5:25.*"
 ARG CONTAINERLAB_VERSION="0.51.*"
 ARG NERDCTL_VERSION="1.7.4"
 
 RUN apt-get update && \
     apt-get install -yq --no-install-recommends \
-        ca-certificates \
-        curl \
-        wget \
-        gnupg \
-        lsb-release \
-        vim \
-        iproute2 \
-        tcpdump \
-        procps \
-        openssh-client \
-        inetutils-ping \
-        traceroute
+            ca-certificates \
+            curl \
+            wget \
+            gnupg \
+            lsb-release \
+            vim \
+            iproute2 \
+            tcpdump \
+            procps \
+            openssh-client \
+            inetutils-ping \
+            traceroute
 
 RUN echo "deb [trusted=yes] https://apt.fury.io/netdevops/ /" | \
     tee -a /etc/apt/sources.list.d/netdevops.list
@@ -65,7 +62,7 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/cache/apt/archive/*.deb
 
-RUN wget -c https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-amd64.tar.gz -O - | tar -xz -C /usr/bin/ && rm /usr/bin/containerd-rootless*.sh
+RUN curl -L https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-amd64.tar.gz | tar -xz -C /usr/bin/ && rm /usr/bin/containerd-rootless*.sh
 
 # https://github.com/docker/cli/issues/4807
 RUN sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker

diff --git a/build/manager.Dockerfile b/build/manager.Dockerfile
@@ -8,19 +8,14 @@ RUN mkdir build
 
 # certificates and subdirs need to be owned by root group for openshift reasons -- otherwise we
 # get permission denied issues when the controller tries to create ca/client subdirs
-RUN mkdir -p certificates/ca \
-RUN mkdir -p mkdir certificates/client
-RUN mkdir -p mkdir certificates/webhook
-RUN chgrp -R root /clabernetes/certificates && \
+RUN mkdir -p certificates/ca && \
+    mkdir -p mkdir certificates/client && \
+    mkdir -p mkdir certificates/webhook && \
+    chgrp -R root /clabernetes/certificates && \
     chmod -R 0770 /clabernetes/certificates
 
-COPY cmd/clabernetes/main.go main.go
-
 COPY . .
 
-COPY go.mod go.mod
-COPY go.sum go.sum
-
 RUN go mod download
 
 RUN CGO_ENABLED=0 \
@@ -32,7 +27,7 @@ RUN CGO_ENABLED=0 \
     -a \
     -o \
     build/manager \
-    main.go
+    cmd/clabernetes/main.go
 
 FROM --platform=linux/amd64 gcr.io/distroless/static-debian12:nonroot
 

diff --git a/clabverter/assets/topology.yaml.template b/clabverter/assets/topology.yaml.template
@@ -56,3 +56,6 @@ spec:
   naming: {{ .Naming }}
   {{- end }}
   connectivity: vxlan
+  definition:
+    containerlab: |-
+      {{- .ClabConfig }}
diff --git a/clabverter/clabverter.go b/clabverter/clabverter.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	sigsyaml "sigs.k8s.io/yaml"
 	"slices"
 	"sort"
 	"strings"
@@ -18,6 +17,7 @@ import (
 	clabernetesutil "github.com/srl-labs/clabernetes/util"
 	clabernetesutilcontainerlab "github.com/srl-labs/clabernetes/util/containerlab"
 	clabernetesutilkubernetes "github.com/srl-labs/clabernetes/util/kubernetes"
+	sigsyaml "sigs.k8s.io/yaml"
 )
 
 const (
@@ -29,7 +29,7 @@ const (
 // MustNewClabverter returns an instance of Clabverter or panics.
 func MustNewClabverter(
 	topologyFile,
-	specsFile,
+	topologySpecFile,
 	outputDirectory,
 	destinationNamespace,
 	naming,
@@ -93,7 +93,7 @@ func MustNewClabverter(
 	return &Clabverter{
 		logger:                  clabverterLogger,
 		topologyFile:            topologyFile,
-		specsFile:               specsFile,
+		topologySpecFile:        topologySpecFile,
 		githubToken:             githubToken,
 		outputDirectory:         outputDirectory,
 		stdout:                  stdout,
@@ -117,7 +117,6 @@ type Clabverter struct {
 	logger claberneteslogging.Instance
 
 	topologyFile    string
-	specsFile       string
 	outputDirectory string
 	stdout          bool
 
@@ -128,10 +127,13 @@ type Clabverter struct {
 
 	disableExpose bool
 
-	topologyPath        string
-	topologyPathParent  string
-	isRemotePath        bool
-	valuesPath          string
+	topologyPath       string
+	topologyPathParent string
+	isRemotePath       bool
+
+	topologySpecFile     string
+	topologySpecFilePath string
+
 	githubGroup         string
 	githubRepo          string
 	githubToken         string
@@ -141,8 +143,6 @@ type Clabverter struct {
 	rawClabConfig string
 	clabConfig    *clabernetesutilcontainerlab.Config
 
-	rawSpecValues string
-
 	// mapping of nodeName -> startup-config info for the templating process; this is its own thing
 	// because configurations may be huge and configmaps have a 1M char limit, so while keeping them
 	// by themselves may not "solve" for ginormous configs, it can certainly give us a little extra
@@ -313,6 +313,19 @@ func (c *Clabverter) load() error {
 		c.topologyPathParent = filepath.Dir(c.topologyPath)
 	}
 
+	if c.topologySpecFile != "" {
+		c.topologySpecFilePath, err = filepath.Abs(c.topologySpecFile)
+		if err != nil {
+			c.logger.Criticalf("failed determining absolute path of values file, error: %s", err)
+
+			return err
+		}
+	}
+
+	c.logger.Debugf(
+		"determined fully qualified topology spec values file path as: %s", c.topologySpecFilePath,
+	)
+
 	c.logger.Debug("attempting to load containerlab topology....")
 
 	var rawClabConfigBytes []byte
@@ -370,37 +383,6 @@ func (c *Clabverter) load() error {
 
 	c.logger.Debug("loading and validating containerlab topology file complete!")
 
-	if c.specsFile != "" {
-		c.valuesPath, err = filepath.Abs(c.specsFile)
-		if err != nil {
-			c.logger.Criticalf("failed determining absolute path of values file, error: %s", err)
-
-			return err
-		}
-
-		c.logger.Debugf(
-			"determined fully qualified spec values file path as: %s", c.valuesPath,
-		)
-
-		c.logger.Debug("attempting to load spec values....")
-
-		rawSpecValuesBytes, err := os.ReadFile(c.valuesPath)
-		if err != nil {
-			c.logger.Criticalf(
-				"failed reading spec values file at '%s' from disk, error: %s",
-				c.valuesPath, err,
-			)
-
-			return err
-		}
-
-		// specs file content is non-indented, need to make it match spec indent
-		c.rawSpecValues = "spec:\n" + clabernetesutil.Indent(
-			string(rawSpecValuesBytes),
-			specIndentSpaces,
-		)
-	}
-
 	return nil
 }
 
@@ -492,8 +474,13 @@ func (c *Clabverter) handleManifest() error {
 	err = t.Execute(
 		&rendered,
 		containerlabTemplateVars{
-			Name:                c.clabConfig.Name,
-			Namespace:           c.destinationNamespace,
+			Name:      c.clabConfig.Name,
+			Namespace: c.destinationNamespace,
+			// pad w/ a newline so the template can look prettier :)
+			ClabConfig: "\n" + clabernetesutil.Indent(
+				c.rawClabConfig,
+				specDefinitionIndentSpaces,
+			),
 			Files:               files,
 			FilesFromURL:        c.extraFilesFromURL,
 			InsecureRegistries:  c.insecureRegistries,
@@ -512,6 +499,8 @@ func (c *Clabverter) handleManifest() error {
 	finalRendered, err := c.mergeConfigSpecWithRenderedTopology(rendered.Bytes())
 	if err != nil {
 		c.logger.Criticalf("failed merging spec config with rendered topology, error: %s", err)
+
+		return err
 	}
 
 	fileName := fmt.Sprintf("%s/%s.yaml", c.outputDirectory, c.clabConfig.Name)
@@ -529,30 +518,46 @@ func (c *Clabverter) handleManifest() error {
 }
 
 func (c *Clabverter) mergeConfigSpecWithRenderedTopology(
-	renderedTopologyBytes []byte,
+	renderedTopologySpecBytes []byte,
 ) ([]byte, error) {
-	finalTopologySpec := &clabernetesapisv1alpha1.Topology{}
+	finalTopology := &clabernetesapisv1alpha1.Topology{}
+
+	if c.topologySpecFilePath == "" {
+		return renderedTopologySpecBytes, nil
+	}
+
+	content, err := os.ReadFile(c.topologySpecFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	topologySpecFromSpecsFile := &clabernetesapisv1alpha1.TopologySpec{}
+
+	err = sigsyaml.Unmarshal(content, topologySpecFromSpecsFile)
+	if err != nil {
+		return nil, err
+	}
 
-	if c.specsFile == "" {
-		return renderedTopologyBytes, nil
+	topologyFromSpecsFile := &clabernetesapisv1alpha1.Topology{
+		Spec: *topologySpecFromSpecsFile,
 	}
 
-	content, err := os.ReadFile(c.specsFile)
+	topologyFromSpecsFileBytes, err := sigsyaml.Marshal(topologyFromSpecsFile)
 	if err != nil {
 		return nil, err
 	}
 
-	err = sigsyaml.Unmarshal(content, &finalTopologySpec)
+	err = sigsyaml.Unmarshal(topologyFromSpecsFileBytes, finalTopology)
 	if err != nil {
 		return nil, err
 	}
 
-	err = sigsyaml.Unmarshal(renderedTopologyBytes, finalTopologySpec)
+	err = sigsyaml.Unmarshal(renderedTopologySpecBytes, finalTopology)
 	if err != nil {
 		return nil, err
 	}
 
-	finalTopologyBytes, err := sigsyaml.Marshal(finalTopologySpec)
+	finalTopologyBytes, err := sigsyaml.Marshal(finalTopology)
 	if err != nil {
 		return nil, err
 	}

diff --git a/clabverter/test-fixtures/clabversiontest/specs.yaml b/clabverter/test-fixtures/clabversiontest/specs.yaml
@@ -1,9 +1,8 @@
 ---
-deployment:
-  statusProbes:
-    enabled: true
-    excludedNodes:
-    - baguette
-    probeConfiguration:
-      tcpProbeConfiguration:
-         port: 22
+statusProbes:
+  enabled: true
+  excludedNodes:
+  - baguette
+  probeConfiguration:
+    tcpProbeConfiguration:
+       port: 22
diff --git a/clabverter/types.go b/clabverter/types.go
@@ -28,6 +28,7 @@ type topologyFileFromURLTemplateVars struct {
 type containerlabTemplateVars struct {
 	Name                string
 	Namespace           string
+	ClabConfig          string
 	Files               map[string][]topologyConfigMapTemplateVars
 	FilesFromURL        map[string][]topologyFileFromURLTemplateVars
 	InsecureRegistries  []string