From 38de02bbf72fb9c2c773e2c70fcf832654de5930 Mon Sep 17 00:00:00 2001 From: "Luis Gustavo S. Barreto" Date: Wed, 19 May 2021 16:45:49 -0300 Subject: [PATCH] gateway: add automated tests We have around 20 location NGINX directives for 5 underlying microservices. This level of complexity slowed down our dev and release process because every change had to be verified manually in a separate test environment. So, we are introducing automated tests for the API gateway. --- gateway/Dockerfile | 26 ++++++++- gateway/shellhub.conf | 45 ++++++++++----- gateway/tests/basic_test.go | 47 +++++++++++++++ gateway/tests/go.mod | 5 ++ gateway/tests/go.sum | 11 ++++ gateway/tests/main_test.go | 112 ++++++++++++++++++++++++++++++++++++ 6 files changed, 229 insertions(+), 17 deletions(-) create mode 100644 gateway/tests/basic_test.go create mode 100644 gateway/tests/go.mod create mode 100644 gateway/tests/go.sum create mode 100644 gateway/tests/main_test.go diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 8524d84f4e3..faadd12f09f 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,4 +1,4 @@ -FROM openresty/openresty:1.15.8.2-6-alpine +FROM openresty/openresty:1.15.8.2-6-alpine as base RUN ["rm", "/etc/nginx/conf.d/default.conf"] @@ -11,6 +11,28 @@ COPY gateway/shellhub.conf /etc/nginx/conf.d/ COPY gateway/kickstart.sh /usr/local/openresty/nginx/html/ COPY gateway/entrypoint.sh / -ENTRYPOINT ["/entrypoint.sh"] +FROM base as test + +RUN apk add --no-cache alpine-sdk + +COPY --from=golang:1.16.4-alpine3.13 /usr/local/go /usr/local/go + +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH + +RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" + +WORKDIR $GOPATH/src/github.com/shellhub-io/shellhub/gateway/tests +COPY ./gateway/tests/go.mod ./gateway/tests/go.sum ./ + +RUN go mod download + +COPY ./gateway/tests $GOPATH/src/github.com/shellhub-io/shellhub/gateway/tests + +CMD ["go", "test"] + +FROM base as production + +ENTRYPOINT ["/entrypoint.sh"] CMD ["/usr/local/openresty/bin/openresty", "-g", "daemon off;"] diff --git a/gateway/shellhub.conf b/gateway/shellhub.conf index 77dec41c5bd..e4d688a3425 100644 --- a/gateway/shellhub.conf +++ b/gateway/shellhub.conf @@ -8,19 +8,22 @@ server { resolver 127.0.0.11 ipv6=off; location / { + set $upstream ui:8080; + add_header Cache-Control "no-cache, no-store"; add_header Pragma "no-cache"; - proxy_pass http://ui:8080; + proxy_pass http://$upstream; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; proxy_http_version 1.1; proxy_cache_bypass $http_upgrade; proxy_redirect off; } location /api { + set $upstream api:8080; + auth_request /auth; auth_request_set $tenant_id $upstream_http_x_tenant_id; auth_request_set $username $upstream_http_x_username; @@ -30,7 +33,7 @@ server { proxy_set_header X-Tenant-ID $tenant_id; proxy_set_header X-Username $username; proxy_set_header X-ID $id; - proxy_pass http://api:8080; + proxy_pass http://$upstream; } {{ if bool (env.Getenv "SHELLHUB_ENTERPRISE") -}} @@ -46,7 +49,6 @@ server { proxy_pass http://$upstream; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; proxy_http_version 1.1; proxy_cache_bypass $http_upgrade; proxy_redirect off; @@ -65,12 +67,13 @@ server { {{ end -}} location /ssh/connection { + set $upstream ssh:8080; + auth_request /auth; auth_request_set $device_uid $upstream_http_x_device_uid; - proxy_pass http://ssh:8080; + proxy_pass http://$upstream; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; {{ if bool (env.Getenv "SHELLHUB_PROXY") -}} proxy_set_header X-Real-IP $proxy_protocol_addr; {{ else -}} @@ -83,10 +86,11 @@ server { } location /ssh/revdial { - proxy_pass http://ssh:8080; + set $upstream ssh:8080; + + proxy_pass http://$upstream; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; {{ if bool (env.Getenv "SHELLHUB_PROXY") -}} proxy_set_header X-Real-IP $proxy_protocol_addr; {{ else -}} @@ -98,10 +102,12 @@ server { } location /ssh/auth { + set $upstream api:8080; + auth_request /auth; auth_request_set $device_uid $upstream_http_x_device_uid; error_page 500 =401 /auth; - proxy_pass http://api:8080; + proxy_pass http://$upstream; proxy_set_header X-Device-UID $device_uid; } @@ -143,37 +149,46 @@ server { {{ end -}} location ~* /api/sessions/(.*)/close { + set $upstream ssh:8080; + auth_request /auth; auth_request_set $tenant_id $upstream_http_x_tenant_id; error_page 500 =401 /auth; rewrite ^/api/(.*)$ /$1 break; proxy_set_header X-Tenant-ID $tenant_id; - proxy_pass http://ssh:8080; + proxy_pass http://$upstream; } location /api/devices/auth { + set $upstream api:8080; + auth_request off; rewrite ^/api/(.*)$ /api/$1 break; - proxy_pass http://api:8080; + proxy_pass http://$upstream; } location /api/login { + set $upstream api:8080; + auth_request off; rewrite ^/api/(.*)$ /api/$1 break; - proxy_pass http://api:8080; + proxy_pass http://$upstream; } location /auth { + set $upstream api:8080; + internal; rewrite ^/(.*)$ /internal/$1 break; - proxy_pass http://api:8080; + proxy_pass http://$upstream; } location /ws { - proxy_pass http://ssh:8080; + set $upstream ssh:8080; + + proxy_pass http://$upstream; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; {{ if bool (env.Getenv "SHELLHUB_PROXY") -}} proxy_set_header X-Real-IP $proxy_protocol_addr; diff --git a/gateway/tests/basic_test.go b/gateway/tests/basic_test.go new file mode 100644 index 00000000000..8a5cbc712eb --- /dev/null +++ b/gateway/tests/basic_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "net/http" + "testing" +) + +func TestDummy(t *testing.T) { + fmt.Println("dummy") + + res := make(chan *http.Request) + + mux := http.NewServeMux() + mux.Handle("/", http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + fmt.Println(r.Header) + fmt.Fprintf(w, "okay") + res <- r + }, + )) + + srv := &http.Server{ + Addr: "127.0.0.1:8080", + Handler: mux, + } + + go func() { + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + panic(err) + } + }() + + if !waitForConnection("tcp", "127.0.0.1:8080") { + panic("Failed to start http server") + } + + client := &http.Client{} + + req, _ := http.NewRequest("GET", "http://localhost:80/", nil) + go client.Do(req) + + rr := <-res + fmt.Println(rr.Host) + + // / => UI +} diff --git a/gateway/tests/go.mod b/gateway/tests/go.mod new file mode 100644 index 00000000000..2a17a596bab --- /dev/null +++ b/gateway/tests/go.mod @@ -0,0 +1,5 @@ +module github.com/shellhub-io/shellhub/gateway/tests + +go 1.16 + +require github.com/miekg/dns v1.1.42 // indirect diff --git a/gateway/tests/go.sum b/gateway/tests/go.sum new file mode 100644 index 00000000000..0004d508ab0 --- /dev/null +++ b/gateway/tests/go.sum @@ -0,0 +1,11 @@ +github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY= +github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 h1:cEhElsAv9LUt9ZUUocxzWe05oFLVd+AA2nstydTeI8g= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/gateway/tests/main_test.go b/gateway/tests/main_test.go new file mode 100644 index 00000000000..33a3db0b21f --- /dev/null +++ b/gateway/tests/main_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "fmt" + "net" + "os" + "os/exec" + "testing" + "time" + + "github.com/miekg/dns" +) + +type dnsHandler struct { + records map[string]string +} + +func (d *dnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { + m := new(dns.Msg) + m.SetReply(r) + m.Compress = false + + switch r.Opcode { + case dns.OpcodeQuery: + d.parseQuery(m) + } + + w.WriteMsg(m) +} + +func (d *dnsHandler) parseQuery(m *dns.Msg) { + for _, q := range m.Question { + switch q.Qtype { + case dns.TypeA: + if ip, ok := d.records[q.Name]; ok { + rr, err := dns.NewRR(fmt.Sprintf("%s A %s", q.Name, ip)) + if err == nil { + m.Answer = append(m.Answer, rr) + } + } + } + } +} + +func waitForConnection(proto, addr string) bool { + for i := 0; i < 10; i++ { + conn, err := net.Dial(proto, addr) + if err == nil { + conn.Close() + return true + } + + time.Sleep(time.Second) + } + + return false +} + +func TestMain(m *testing.M) { + // Create a virtual network adapter + if _, err := exec.Command("ifconfig", "eth0:0", "127.0.0.11").Output(); err != nil { + panic(err) + } + + server := &dns.Server{ + Addr: "127.0.0.11:53", + Net: "udp", + Handler: &dnsHandler{ + records: map[string]string{ + "api.": "127.0.0.1", + "ssh.": "127.0.0.1", + "ui.": "127.0.0.1", + }, + }, + } + + go func() { + if err := server.ListenAndServe(); err != nil { + panic(err) + } + }() + + // Wait for DNS test server to be started + if !waitForConnection("udp", "127.0.0.11:53") { + panic("Failed to connect to DNS test server") + } + + // Start OpenResty daemon + cmd := exec.Command("/entrypoint.sh", "/usr/local/openresty/bin/openresty", "-g", "daemon off;") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Start(); err != nil { + panic(err) + } + + // Wait for OpenResty to be started + if !waitForConnection("tcp", "127.0.0.1:80") { + panic("Failed to connect to OpenResty") + } + + // Run unit test + code := m.Run() + + server.Shutdown() + + if err := cmd.Process.Kill(); err != nil { + panic(err) + } + + os.Exit(code) +}