Add higress e2e test (#38)

.github/workflows/e2e.yaml

@@ -18,6 +18,7 @@ jobs:
         target:
           - coraza
           - envoyDispatchCall
+          - higressGCTest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v4

e2e/higress-gc-test/docker-compose.yml

@@ -0,0 +1,12 @@
+services:
+  envoy:
+    image: ${ENVOY_IMAGE:-envoyproxy/envoy:v1.28-latest}
+    command:
+      - -c
+      - /conf/envoy-config.yaml
+    volumes:
+      - ./build:/build
+      - .:/conf
+    ports:
+    - 8080:8080
+    - 8082:8082

e2e/higress-gc-test/envoy-config.yaml

@@ -0,0 +1,58 @@
+node:
+  cluster: test-cluster
+  id: test-idn
+
+admin:
+  address:
+    socket_address: { address: 0.0.0.0, port_value: 8082 }
+
+static_resources:
+  listeners:
+    - name: listener_0
+      address:
+        socket_address: { address: 0.0.0.0, port_value: 8080 }
+      per_connection_buffer_limit_bytes: 1024000000
+      filter_chains:
+        - filters:
+            - name: envoy.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                stat_prefix: ingress_http
+                access_log:
+                  - name: envoy.access_loggers.stdout
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
+                http_filters:
+                  - name: gctest
+                    typed_config:
+                      "@type": type.googleapis.com/udpa.type.v1.TypedStruct
+                      type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
+                      value:
+                        config:
+                          name: basic_auth
+                          vm_config:
+                            runtime: envoy.wasm.runtime.v8
+                            code:
+                              local:
+                                filename: /build/plugin.wasm
+                            allow_precompiled: true
+                          fail_open: true
+                          configuration:
+                            "@type": type.googleapis.com/google.protobuf.StringValue
+                            value: '10485760'
+                  - name: envoy.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+                route_config:
+                  name: local_route
+                  virtual_hosts:
+                    - name: local_service
+                      domains:
+                        - "*"
+                      routes:
+                        - match:
+                            prefix: /
+                          direct_response:
+                            status: 200
+                            body:
+                              inline_string: "hello world"

e2e/higress-gc-test/go.mod

@@ -0,0 +1,7 @@
+module higress-gc-test
+
+go 1.20
+
+require github.com/tetratelabs/proxy-wasm-go-sdk v0.19.1-0.20220822060051-f9d179a57f8c
+
+require github.com/stretchr/testify v1.8.0 // indirect

e2e/higress-gc-test/go.sum

@@ -0,0 +1,16 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/tetratelabs/proxy-wasm-go-sdk v0.19.1-0.20220822060051-f9d179a57f8c h1:OCUFXVGixHLfNjg6/QYEhv+jHJ5mRGhpEUVFv9eWPJE=
+github.com/tetratelabs/proxy-wasm-go-sdk v0.19.1-0.20220822060051-f9d179a57f8c/go.mod h1:5t/pWFNJ9eMyu/K/Z+OeGhDJ9sN9eCo8fc2pyM/Qjg4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

e2e/higress-gc-test/main.go

@@ -0,0 +1,72 @@
+// Copyright wasilibs authors
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"runtime"
+	"strconv"
+
+	"github.com/tetratelabs/proxy-wasm-go-sdk/proxywasm"
+	"github.com/tetratelabs/proxy-wasm-go-sdk/proxywasm/types"
+
+	_ "github.com/wasilibs/nottinygc"
+)
+
+func main() {
+	proxywasm.SetVMContext(&vm{})
+}
+
+type vm struct {
+	types.DefaultVMContext
+}
+
+func (v *vm) NewPluginContext(contextID uint32) types.PluginContext {
+	return &plugin{}
+}
+
+type plugin struct {
+	// Embed the default plugin context here,
+	// so that we don't need to reimplement all the methods.
+	types.DefaultPluginContext
+
+	size int
+}
+
+// OnPluginStart Override types.DefaultPluginContext.
+func (h *plugin) OnPluginStart(_ int) types.OnPluginStartStatus {
+	data, err := proxywasm.GetPluginConfiguration()
+	if err != nil {
+		panic(err)
+	}
+	sz, err := strconv.Atoi(string(bytes.TrimSpace(data)))
+	if err != nil {
+		panic(err)
+	}
+	h.size = sz
+	return types.OnPluginStartStatusOK
+}
+
+// NewHttpContext Override types.DefaultPluginContext to allow us to declare a request handler for each
+// intercepted request the Envoy Sidecar sends us
+func (h *plugin) NewHttpContext(_ uint32) types.HttpContext {
+	return &tester{size: h.size}
+}
+
+type tester struct {
+	types.DefaultHttpContext
+	size int
+}
+
+func (c *tester) OnHttpRequestHeaders(numHeaders int, endOfStream bool) types.Action {
+	b := make([]byte, c.size)
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	proxywasm.LogInfof("alloc success, point address: %p", b)
+	memstats := fmt.Sprintf(`{"Sys": %d,"HeapSys": %d,"HeapIdle": %d,"HeapInuse": %d,"HeapReleased": %d}`, m.Sys, m.HeapSys, m.HeapIdle, m.HeapInuse, m.HeapReleased)
+	proxywasm.LogInfo(memstats)
+	proxywasm.SendHttpResponse(200, [][2]string{{"Content-Type", "application/json"}}, []byte(memstats), -1)
+	return types.ActionContinue
+}

go.work

@@ -6,4 +6,5 @@ use (
 	.
 	./bench
 	./e2e/envoy-dispatch-call
+	./e2e/higress-gc-test
 )

magefiles/e2e.go

@@ -18,10 +18,6 @@ import (
 )
 
 func E2eCoraza() error {
-	if err := os.MkdirAll(filepath.Join("build", "logs"), 0o755); err != nil {
-		return err
-	}
-
 	if _, err := os.Stat(filepath.Join("e2e", "coraza-proxy-wasm")); os.IsNotExist(err) {
 		// Try not pinning version, there should be no compatibility issues causing unexpected failures from a
 		// green coraza build so we get to keep forward coverage this way.
@@ -66,10 +62,6 @@ func E2eCoraza() error {
 }
 
 func E2eEnvoyDispatchCall() error {
-	if err := os.MkdirAll(filepath.Join("build", "logs"), 0o755); err != nil {
-		return err
-	}
-
 	if err := os.MkdirAll(filepath.Join("e2e", "envoy-dispatch-call", "build"), 0o755); err != nil {
 		return err
 	}
@@ -88,7 +80,7 @@ func E2eEnvoyDispatchCall() error {
 		}
 	}()
 
-	stats, err := e2eLoad("http://localhost:8080/status/200", "http://localhost:8082/stats")
+	stats, err := e2eLoad("http://localhost:8080/status/200", "http://localhost:8082/stats", 40, 5000)
 	if err != nil {
 		return err
 	}
@@ -120,6 +112,53 @@ func E2eEnvoyDispatchCall() error {
 	return nil
 }
 
+func E2eHigressGCTest() error {
+	if err := os.MkdirAll(filepath.Join("e2e", "higress-gc-test", "build"), 0o755); err != nil {
+		return err
+	}
+
+	if err := sh.RunV("tinygo", "build", "-target=wasi", "-gc=custom", "-tags='custommalloc nottinygc_envoy'", "-scheduler=none",
+		"-o", filepath.Join("e2e", "higress-gc-test", "build", "plugin.wasm"), "./e2e/higress-gc-test"); err != nil {
+		return err
+	}
+
+	if err := sh.RunV("docker-compose", "--file", filepath.Join("e2e", "higress-gc-test", "docker-compose.yml"), "up", "-d"); err != nil {
+		return err
+	}
+	defer func() {
+		if err := sh.RunV("docker-compose", "--file", filepath.Join("e2e", "higress-gc-test", "docker-compose.yml"), "down", "-v"); err != nil {
+			panic(err)
+		}
+	}()
+
+	_, err := e2eLoad("http://localhost:8080/hello", "http://localhost:8082/stats", 2, 10000)
+	if err != nil {
+		return err
+	}
+
+	type memStats struct {
+		Sys int `json:"Sys"`
+	}
+
+	res, err := http.Get("http://localhost:8080/hello")
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	var stats memStats
+	if err := json.NewDecoder(res.Body).Decode(&stats); err != nil {
+		return err
+	}
+
+	// We expect around 20MB per VM (this reports per VM stat), a conservative
+	// 100MB should be a fine check without flakiness
+	if mem := stats.Sys; mem > 100_000_000 {
+		return fmt.Errorf("expected <100MB memory used, actual: %d", mem)
+	}
+
+	return nil
+}
+
 type counterStat struct {
 	Name  string `json:"name"`
 	Value int    `json:"value"`
@@ -130,7 +169,7 @@ type counterStats struct {
 }
 
 // If needed, we can try being more sophisticated later but run some simple load for now.
-func e2eLoad(url string, statsURL string) (*counterStats, error) {
+func e2eLoad(url string, statsURL string, p int, n int) (*counterStats, error) {
 	wg := sync.WaitGroup{}
 
 	var success atomic.Uint32
@@ -154,8 +193,6 @@ func e2eLoad(url string, statsURL string) (*counterStats, error) {
 		return nil, errors.New("failed to get healthy in 100 attempts")
 	}
 
-	p := 40
-	n := 5000
 	for i := 0; i < p; i++ {
 		wg.Add(1)
 		go func() {
@@ -193,3 +230,9 @@ func e2eLoad(url string, statsURL string) (*counterStats, error) {
 
 	return &stats, nil
 }
+
+func init() {
+	if err := os.MkdirAll(filepath.Join("build", "logs"), 0o755); err != nil {
+		panic(err)
+	}
+}