Uses go:embed for wasm stats and helm chart

The way we populate the value for wasm stats and helm charts is by using the linker -X flag. This makes the linking command to be extremely long (40k+ chars). This is above the ARG_MAX limit posed by cygwin. Fixes #3573 This patch updates to go version 1.16 and uses go:embed to embed the files into variables Co-authored-by: Jon Huhn [email protected] Signed-off-by: Sotiris Nanopoulos <[email protected]>
openservicemesh · Jun 10, 2021 · a8daa83 · a8daa83
1 parent 9711165
commit a8daa83
Show file tree

Hide file tree

Showing 17 changed files with 100 additions and 125 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -40,10 +40,17 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v1
+      - name: Setup Go 1.16
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.16
+      - name: go build deps
+        run: make embed-files-test
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
           version: v1.30
+          skip-pkg-cache: true
           args: --timeout 5m --skip-dirs pkg/gen --skip-files "zz_generated.deepcopy.go$"
 
   tidy:
@@ -52,6 +59,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v1
+      - name: Setup Go 1.16
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.16
       - name: go mod tidy
         run: make go-mod-tidy
 
@@ -91,12 +102,12 @@ jobs:
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-gobuild-${{ hashFiles('**/*.go') }}
-      - name: Setup Go 1.15
+      - name: Setup Go 1.16
         uses: actions/setup-go@v1
         with:
-          go-version: 1.15
+          go-version: 1.16
       - name: Go Build
-        run: go build -v ./...
+        run: make build-ci
 
   unittest:
     name: Go test
@@ -117,10 +128,10 @@ jobs:
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-gobuild-${{ hashFiles('**/*.go') }}
-      - name: Setup Go 1.15
+      - name: Setup Go 1.16
         uses: actions/setup-go@v1
         with:
-          go-version: 1.15
+          go-version: 1.16
       - name: Test
         run: make go-test-coverage
       - name: Upload Coverage
@@ -146,10 +157,10 @@ jobs:
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-gobuild-${{ hashFiles('**/*.go') }}
-      - name: Setup Go 1.15
+      - name: Setup Go 1.16
         uses: actions/setup-go@v1
         with:
-          go-version: 1.15
+          go-version: 1.16
       - name: Test
         run: |
           touch .env
@@ -181,10 +192,10 @@ jobs:
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-gobuild-${{ hashFiles('**/*.go') }}
-      - name: Setup Go 1.15
+      - name: Setup Go 1.16
         uses: actions/setup-go@v1
         with:
-          go-version: 1.15
+          go-version: 1.16
       - name: Build test dependencies
         run: make docker-build-init-osm-controller docker-build-osm-controller docker-build-osm-injector docker-build-init build-osm docker-build-tcp-echo-server
       # PR Tests
@@ -270,10 +281,10 @@ jobs:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-gobuild-${{ hashFiles('**/*.go') }}
 
-      - name: Setup Go 1.15
+      - name: Setup Go 1.16
         uses: actions/setup-go@v1
         with:
-          go-version: 1.15
+          go-version: 1.16
         id: go
 
       - name: Run Simulation w/ Tresor, SMI policies, and egress disabled
@@ -313,10 +324,10 @@ jobs:
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-gobuild-${{ hashFiles('**/*.go') }}
-      - name: Setup Go 1.15
+      - name: Setup Go 1.16
         uses: actions/setup-go@v1
         with:
-          go-version: 1.15
+          go-version: 1.16
       - name: Docker Login
         run: docker login --username "$DOCKER_USER" --password-stdin <<< "$DOCKER_PASS"
       - name: Push images with git sha tag

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -39,10 +39,10 @@ jobs:
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-gobuild-${{ hashFiles('**/*.go') }}
-      - name: Setup Go 1.15
+      - name: Setup Go 1.16
         uses: actions/setup-go@v1
         with:
-          go-version: 1.15
+          go-version: 1.16
       - name: Build Binaries
         run: |
           make release-artifacts
@@ -128,10 +128,10 @@ jobs:
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-gobuild-${{ hashFiles('**/*.go') }}
-      - name: Setup Go 1.15
+      - name: Setup Go 1.16
         uses: actions/setup-go@v1
         with:
-          go-version: 1.15
+          go-version: 1.16
       - name: Docker Login
         run: docker login --username "$DOCKER_USER" --password-stdin <<< "$DOCKER_PASS"
       - name: Push images with version tag

diff --git a/.gitignore b/.gitignore
@@ -44,6 +44,7 @@ _dist/
 creds.json
 screenlog.*
 demo/bin/
+*.tgz
 *.wasm
 
 ## Ignore test coverage files

diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 
 #build stage
-FROM golang:1.15-alpine AS builder
+FROM golang:1.16-alpine AS builder
 
 RUN apk update
 RUN apk add --no-cache make

diff --git a/Makefile b/Makefile
@@ -1,7 +1,6 @@
 #!make
 
 TARGETS         := darwin/amd64 linux/amd64 windows/amd64
-SHELL           := bash -o pipefail
 BINNAME         ?= osm
 DIST_DIRS       := find * -type d -exec
 CTR_REGISTRY    ?= openservicemesh
@@ -18,7 +17,7 @@ BUILD_DATE_VAR := github.com/openservicemesh/osm/pkg/version.BuildDate
 BUILD_VERSION_VAR := github.com/openservicemesh/osm/pkg/version.Version
 BUILD_GITCOMMIT_VAR := github.com/openservicemesh/osm/pkg/version.GitCommit
 
-LDFLAGS ?= "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(VERSION) -X $(BUILD_GITCOMMIT_VAR)=$(GIT_SHA) -X main.chartTGZSource=$$(cat -) -s -w"
+LDFLAGS ?= "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(VERSION) -X $(BUILD_GITCOMMIT_VAR)=$(GIT_SHA) -s -w"
 
 # These two values are combined and passed to go test
 E2E_FLAGS ?= -installType=KindCluster
@@ -34,37 +33,6 @@ ifeq ($(GO_VERSION_PATCH),)
 GO_VERSION_PATCH := 0
 endif
 
-# Required Go version
-# These variables set the minimum required version of Go for this project.
-# The minimum version below is defined based on:
-#   - required language features
-#   - known required bug fixes
-#   - CVEs
-# Current: 1.15.7
-# For more context see https://github.com/openservicemesh/osm/issues/2363
-# For CVEs fixed with 1.15.7 see https://groups.google.com/g/golang-announce/c/mperVMGa98w
-MIN_REQUIRED_GO_VERSION_MAJOR = 1
-MIN_REQUIRED_GO_VERSION_MINOR = 15
-MIN_REQUIRED_GO_VERSION_PATCH = 7
-
-GO_VERSION_MESSAGE = "Installed Go version is $(GO_VERSION_MAJOR).$(GO_VERSION_MINOR).$(GO_VERSION_PATCH).\n OSM requires Go version $(MIN_REQUIRED_GO_VERSION_MAJOR).$(MIN_REQUIRED_GO_VERSION_MINOR).$(MIN_REQUIRED_GO_VERSION_PATCH) or higher!\n\n"
-
-check-go-version: # Ensure the Go version used is what OSM requires
-	@if [ $(GO_VERSION_MAJOR) -gt $(MIN_REQUIRED_GO_VERSION_MAJOR) ]; then \
-		exit 0 ;\
-	elif [ $(GO_VERSION_MAJOR) -lt $(MIN_REQUIRED_GO_VERSION_MAJOR) ]; then \
-		echo -e '$(GO_VERSION_MESSAGE)';\
-		exit 1; \
-	elif [ $(GO_VERSION_MINOR) -gt $(MIN_REQUIRED_GO_VERSION_MINOR) ] ; then \
-		exit 0; \
-	elif [ $(GO_VERSION_MINOR) -lt $(MIN_REQUIRED_GO_VERSION_MINOR) ] ; then \
-		echo -e '$(GO_VERSION_MESSAGE)';\
-		exit 1; \
-	elif [ $(GO_VERSION_PATCH) -lt $(MIN_REQUIRED_GO_VERSION_PATCH) ] ; then \
-		echo -e '$(GO_VERSION_MESSAGE)';\
-		exit 1; \
-	fi
-
 check-env:
 ifndef CTR_REGISTRY
 	$(error CTR_REGISTRY environment variable is not defined; see the .env.example file for more information; then source .env)
@@ -93,20 +61,23 @@ clean-osm-injector:
 build: build-init-osm-controller build-osm-controller build-osm-injector
 
 .PHONY: build-init-osm-controller
-build-init-osm-controller: check-go-version clean-init-osm-controller wasm/stats.wasm
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./bin/init-osm-controller/init-osm-controller -ldflags "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(VERSION) -X $(BUILD_GITCOMMIT_VAR)=$(GIT_SHA) -X github.com/openservicemesh/osm/pkg/envoy/lds.statsWASMBytes=$$(base64 < wasm/stats.wasm | tr -d \\n) -s -w" ./cmd/init-osm-controller
+build-init-osm-controller: clean-init-osm-controller
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./bin/init-osm-controller/init-osm-controller -ldflags ${LDFLAGS} ./cmd/init-osm-controller
 
 .PHONY: build-osm-controller
-build-osm-controller: check-go-version clean-osm-controller wasm/stats.wasm
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./bin/osm-controller/osm-controller -ldflags "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(VERSION) -X $(BUILD_GITCOMMIT_VAR)=$(GIT_SHA) -X github.com/openservicemesh/osm/pkg/envoy/lds.statsWASMBytes=$$(base64 < wasm/stats.wasm | tr -d \\n) -s -w" ./cmd/osm-controller
+build-osm-controller: clean-osm-controller pkg/envoy/lds/stats.wasm
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./bin/osm-controller/osm-controller -ldflags ${LDFLAGS} ./cmd/osm-controller
 
 .PHONY: build-osm-injector
-build-osm-injector: check-go-version clean-osm-injector
+build-osm-injector: clean-osm-injector
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./bin/osm-injector/osm-injector -ldflags "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(VERSION) -X $(BUILD_GITCOMMIT_VAR)=$(GIT_SHA) -s -w" ./cmd/osm-injector
 
 .PHONY: build-osm
-build-osm: check-go-version
-	go run scripts/generate_chart/generate_chart.go | CGO_ENABLED=0  go build -v -o ./bin/osm -ldflags ${LDFLAGS} ./cmd/cli
+build-osm: cmd/cli/chart.tgz
+	CGO_ENABLED=0  go build -v -o ./bin/osm -ldflags ${LDFLAGS} ./cmd/cli
+
+cmd/cli/chart.tgz: scripts/generate_chart/generate_chart.go $(wildcard charts/osm/**/*)
+	go run $< > $@
 
 .PHONY: clean-osm
 clean-osm:
@@ -144,7 +115,7 @@ go-vet:
 	go vet ./...
 
 .PHONY: go-lint
-go-lint:
+go-lint: cmd/cli/chart.tgz pkg/envoy/lds/stats.wasm
 	go run github.com/golangci/golangci-lint/cmd/golangci-lint run --config .golangci.yml
 
 .PHONY: go-fmt
@@ -160,7 +131,7 @@ go-test:
 	./scripts/go-test.sh
 
 .PHONY: go-test-coverage
-go-test-coverage:
+go-test-coverage: embed-files
 	./scripts/test-w-coverage.sh
 
 .PHONY: kind-up
@@ -215,12 +186,23 @@ docker-build-osm-controller: build-osm-controller
 docker-build-osm-injector: build-osm-injector
 	docker build -t $(CTR_REGISTRY)/osm-injector:$(CTR_TAG) -f dockerfiles/Dockerfile.osm-injector bin/osm-injector
 
-wasm/stats.wasm: wasm/stats.cc wasm/Makefile
+pkg/envoy/lds/stats.wasm: wasm/stats.cc wasm/Makefile
 	docker run --rm -v $(PWD)/wasm:/work -w /work openservicemesh/proxy-wasm-cpp-sdk:956f0d500c380cc1656a2d861b7ee12c2515a664 /build_wasm.sh
-
+	@mv wasm/stats.wasm $@
 .PHONY: docker-build
 docker-build: $(DOCKER_DEMO_TARGETS) docker-build-init docker-build-init-osm-controller  docker-build-osm-controller docker-build-osm-injector
 
+.PHONY: embed-files
+embed-files: cmd/cli/chart.tgz pkg/envoy/lds/stats.wasm
+
+.PHONY: embed-files-test
+embed-files-test:
+	./scripts/generate-dummy-embed.sh
+
+.PHONY: build-ci
+build-ci: embed-files
+	go build -v ./...
+
 # docker-push-bookbuyer, etc
 DOCKER_PUSH_TARGETS = $(addprefix docker-push-, $(DEMO_TARGETS) init init-osm-controller osm-controller osm-injector)
 VERIFY_TAGS = 0
@@ -245,8 +227,8 @@ install-git-pre-push-hook:
 # -------------------------------------------
 
 .PHONY: build-cross
-build-cross:
-	go run scripts/generate_chart/generate_chart.go | GO111MODULE=on CGO_ENABLED=0 $(GOX) -ldflags $(LDFLAGS) -parallel=3 -output="_dist/{{.OS}}-{{.Arch}}/$(BINNAME)" -osarch='$(TARGETS)' ./cmd/cli
+build-cross: cmd/cli/chart.tgz
+	GO111MODULE=on CGO_ENABLED=0 $(GOX) -ldflags $(LDFLAGS) -parallel=3 -output="_dist/{{.OS}}-{{.Arch}}/$(BINNAME)" -osarch='$(TARGETS)' ./cmd/cli
 
 .PHONY: dist
 dist:

diff --git a/cmd/cli/install.go b/cmd/cli/install.go
@@ -1,7 +1,10 @@
 package main
 
 import (
+	"bytes"
 	"context"
+
+	_ "embed" // required to embed resources
 	"fmt"
 	"io"
 	"time"
@@ -18,7 +21,6 @@ import (
 	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
-	"github.com/openservicemesh/osm/pkg/cli"
 	"github.com/openservicemesh/osm/pkg/constants"
 )
 
@@ -58,9 +60,10 @@ const (
 	defaultEnforceSingleMesh = false
 )
 
-// chartTGZSource is a base64-encoded, gzipped tarball of the default Helm chart.
-// Its value is initialized at build time.
-var chartTGZSource string
+// chartTGZSource is the `helm package`d representation of the default Helm chart.
+// Its value is embedded at build time.
+//go:embed chart.tgz
+var chartTGZSource []byte
 
 type installCmd struct {
 	out            io.Writer
@@ -141,7 +144,7 @@ func (i *installCmd) loadOSMChart() error {
 	if i.chartPath != "" {
 		i.chartRequested, err = loader.Load(i.chartPath)
 	} else {
-		i.chartRequested, err = cli.LoadChart(chartTGZSource)
+		i.chartRequested, err = loader.LoadArchive(bytes.NewReader(chartTGZSource))
 	}
 
 	if err != nil {

diff --git a/cmd/cli/mesh_upgrade.go b/cmd/cli/mesh_upgrade.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"time"
@@ -10,8 +11,6 @@ import (
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/chartutil"
-
-	"github.com/openservicemesh/osm/pkg/cli"
 )
 
 const (
@@ -30,15 +29,15 @@ overridden from the default for the "osm install" command, the --mesh-name and
 Values from the current Helm release will be carried over to the new release
 with the exception of OpenServiceMesh.image.registry (--container-registry) and
 OpenServiceMesh.image.tag (--osm-image-tag), which will be overridden from the
-old release by default. 
+old release by default.
 
 Note: edits to resources NOT made by Helm or the OSM CLI may not persist after
 "osm mesh upgrade" is run.
 
 Note: edits made to chart values that impact the preset-mesh-config will not
-apply to the osm-mesh-config, when "osm mesh upgrade" is run. This means configuration 
+apply to the osm-mesh-config, when "osm mesh upgrade" is run. This means configuration
 changes made to the osm-mesh-config resource will persist through an upgrade
-and any configuration changes needed can be done by patching this resource prior or 
+and any configuration changes needed can be done by patching this resource prior or
 post an upgrade.
 
 If any CustomResourceDefinitions (CRDs) are different between the installed
@@ -48,7 +47,7 @@ updating the mesh to ensure compatibility.
 `
 
 const meshUpgradeExample = `
-# Upgrade the mesh with the default name in the osm-system namespace, setting 
+# Upgrade the mesh with the default name in the osm-system namespace, setting
 # the image registry and tag to the defaults, and leaving all other values unchanged.
 osm mesh upgrade --osm-namespace osm-system
 `
@@ -100,7 +99,7 @@ func newMeshUpgradeCmd(config *helm.Configuration, out io.Writer) *cobra.Command
 func (u *meshUpgradeCmd) run(config *helm.Configuration) error {
 	if u.chart == nil {
 		var err error
-		u.chart, err = cli.LoadChart(chartTGZSource)
+		u.chart, err = loader.LoadArchive(bytes.NewReader(chartTGZSource))
 		if err != nil {
 			return err
 		}