From f48f53de05e7cb04fa558851bf9d09628d204381 Mon Sep 17 00:00:00 2001 From: Fabio Cicerchia Date: Fri, 4 Dec 2020 19:53:43 +0100 Subject: [PATCH] fix: fixing to achieve good score on BetterCodeHub - #40 --- .github/workflows/main.yml | 12 +- Makefile | 14 +- README.md | 24 +- cache/cache.go | 85 ++++-- cache/engine/client/client.go | 32 +- cache/engine/client/client_test.go | 49 +-- config/config.go | 368 +++++++++-------------- go.mod | 3 +- go.sum | 20 -- main.go | 3 +- server/handler/handler.go | 103 +++++++ server/handler/healthcheck.go | 6 +- server/handler/healthcheck_test.go | 9 +- server/handler/http.go | 176 ++++------- server/handler/http_functional_test.go | 27 +- server/handler/http_unit_test.go | 27 +- server/handler/purge.go | 27 +- server/handler/purge_test.go | 9 +- server/handler/redirect.go | 8 +- server/logger/log.go | 3 +- server/response/response.go | 8 +- server/server.go | 104 ++++--- server/storage/storage.go | 73 ++--- server/tls/tls.go | 28 +- server/transport/http.go | 31 +- utils/{ => base64}/base64.go | 10 +- utils/{ => base64}/base64_test.go | 16 +- utils/circuit-breaker/circuit-breaker.go | 62 ++++ utils/{ => convert}/convert.go | 16 +- utils/convert/convert_test.go | 84 ++++++ utils/convert_test.go | 102 ------- utils/{ => msgpack}/msgpack.go | 10 +- utils/{ => msgpack}/msgpack_test.go | 12 +- utils/{ => slice}/slice.go | 2 +- utils/{ => slice}/slice_test.go | 58 +--- utils/{ => ttl}/ttl.go | 63 ++-- utils/{ => ttl}/ttl_test.go | 40 +-- utils/utils_test.go | 2 +- 38 files changed, 894 insertions(+), 832 deletions(-) create mode 100644 server/handler/handler.go rename utils/{ => base64}/base64.go (77%) rename utils/{ => base64}/base64_test.go (73%) create mode 100644 utils/circuit-breaker/circuit-breaker.go rename utils/{ => convert}/convert.go (68%) create mode 100644 utils/convert/convert_test.go delete mode 100644 utils/convert_test.go rename utils/{ => msgpack}/msgpack.go (79%) rename utils/{ => msgpack}/msgpack_test.go (76%) rename utils/{ => slice}/slice.go (99%) rename utils/{ => slice}/slice_test.go (71%) rename utils/{ => ttl}/ttl.go (56%) rename utils/{ => ttl}/ttl_test.go (67%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e208aef7..3e30b58e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,13 +28,13 @@ jobs: ./go-proxy-cache -debug -config=config.sample.yml & make test + - name: Coverage + run: make cover + - name: Codecov - run: | - redis-server & - ./bin/gen-selfsigned-cert.sh - ./go-proxy-cache -debug -config=config.sample.yml & - make cover - make codecov + run: make codecov + + - uses: paambaati/codeclimate-action@latest sca: runs-on: ubuntu-20.04 diff --git a/Makefile b/Makefile index 93dc95a0..80d3d461 100644 --- a/Makefile +++ b/Makefile @@ -90,10 +90,16 @@ test-functional: ## test functional test-endtoend: ## test endtoend go test -race -count=1 --tags=endtoend ./... -cover: ## coverage - go test -race -count=1 --tags=all -coverprofile cover.out ./... - go tool cover -func=cover.out - go tool cover -html=cover.out +.codeclimate: + curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter + chmod +x ./cc-test-reporter + +cover: .codeclimate ## coverage + # ./cc-test-reporter before-build + go test -race -count=1 --tags=all -coverprofile c.out ./... + go tool cover -func=c.out + go tool cover -html=c.out + # ./cc-test-reporter after-build codecov: ## codecov curl -s https://codecov.io/bash | bash diff --git a/README.md b/README.md index 028f9e40..c70babf2 100644 --- a/README.md +++ b/README.md @@ -7,20 +7,30 @@ Simple Reverse Proxy with Caching, written in Go, using Redis. >>> **(semi) production-ready** <<< -[![MIT License](https://img.shields.io/badge/License-MIT-lightgrey.svg?longCache=true)](LICENSE) +[![MIT License](https://img.shields.io/badge/License-MIT-brightgreen.svg?longCache=true)](LICENSE) [![Pull Requests](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?longCache=true)](https://github.com/fabiocicerchia/go-proxy-cache/pulls) +![Maintenance](https://img.shields.io/maintenance/yes/2020) +[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go) + ![Last Commit](https://img.shields.io/github/last-commit/fabiocicerchia/go-proxy-cache) ![Release Date](https://img.shields.io/github/release-date/fabiocicerchia/go-proxy-cache) +![GitHub all releases](https://img.shields.io/github/downloads/fabiocicerchia/go-proxy-cache/total) -![Docker pulls](https://img.shields.io/docker/pulls/fabiocicerchia/go-proxy-cache.svg "Docker pulls") -![Docker stars](https://img.shields.io/docker/stars/fabiocicerchia/go-proxy-cache.svg "Docker stars") +![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/fabiocicerchia/go-proxy-cache) +![GitHub release (latest by date)](https://img.shields.io/github/v/release/fabiocicerchia/go-proxy-cache) + +![Docker pulls](https://img.shields.io/docker/pulls/fabiocicerchia/go-proxy-cache "Docker pulls") +![Docker stars](https://img.shields.io/docker/stars/fabiocicerchia/go-proxy-cache "Docker stars") + +![GitHub Workflow Status](https://img.shields.io/github/workflow/status/fabiocicerchia/go-proxy-cache/Builds) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4469/badge)](https://bestpractices.coreinfrastructure.org/projects/4469) -[![Go Report Card](https://goreportcard.com/badge/github.com/fabiocicerchia/go-proxy-cache)](https://goreportcard.com/report/github.com/fabiocicerchia/go-proxy-cache) +[![BCH compliance](https://bettercodehub.com/edge/badge/fabiocicerchia/go-proxy-cache?branch=main)](https://bettercodehub.com/results/fabiocicerchia/go-proxy-cache) +[![Go Report Card](https://goreportcard.com/badge/github.com/fabiocicerchia/go-proxy-cache)](https://goreportcard.com/report/github.com/fabiocicerchia/go-proxy-cache) [![codecov](https://codecov.io/gh/fabiocicerchia/go-proxy-cache/branch/main/graph/badge.svg)](https://codecov.io/gh/fabiocicerchia/go-proxy-cache) -[![Maintainability](https://api.codeclimate.com/v1/badges/6cf8c9ea02b75fccf8b5/maintainability)](https://codeclimate.com/github/fabiocicerchia/go-proxy-cache/maintainability) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/fabiocicerchia/go-proxy-cache.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/fabiocicerchia/go-proxy-cache/alerts/) -![Builds](https://github.com/fabiocicerchia/go-proxy-cache/workflows/Builds/badge.svg) +[![Maintainability](https://img.shields.io/codeclimate/maintainability/fabiocicerchia/go-proxy-cache)](https://codeclimate.com/github/fabiocicerchia/go-proxy-cache/maintainability) +[![Technical Debt](https://img.shields.io/codeclimate/tech-debt/fabiocicerchia/go-proxy-cache)](https://codeclimate.com/github/fabiocicerchia/go-proxy-cache/maintainability) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/fabiocicerchia/go-proxy-cache.svg)](https://lgtm.com/projects/g/fabiocicerchia/go-proxy-cache/alerts/) diff --git a/cache/cache.go b/cache/cache.go index 811e76dc..7b0148e9 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -18,11 +18,18 @@ import ( "time" "github.com/fabiocicerchia/go-proxy-cache/cache/engine" - "github.com/fabiocicerchia/go-proxy-cache/config" "github.com/fabiocicerchia/go-proxy-cache/utils" + "github.com/fabiocicerchia/go-proxy-cache/utils/slice" log "github.com/sirupsen/logrus" ) +// CacheObj - Contains cache settings and current cached/cacheable object. +type CacheObj struct { + AllowedStatuses []int + AllowedMethods []string + CurrentObj URIObj +} + // URIObj - Holds details about the response type URIObj struct { URL url.URL @@ -35,51 +42,73 @@ type URIObj struct { } // IsStatusAllowed - Checks if a status code is allowed to be cached. -func IsStatusAllowed(statusCode int) bool { - return utils.ContainsInt(config.Config.Cache.AllowedStatuses, statusCode) +func (c CacheObj) IsStatusAllowed() bool { + return slice.ContainsInt(c.AllowedStatuses, c.CurrentObj.StatusCode) } // IsMethodAllowed - Checks if a HTTP method is allowed to be cached. -func IsMethodAllowed(method string) bool { - return utils.ContainsString(config.Config.Cache.AllowedMethods, method) +func (c CacheObj) IsMethodAllowed() bool { + return slice.ContainsString(c.AllowedMethods, c.CurrentObj.Method) } -// StoreFullPage - Stores the whole page response in cache. -func StoreFullPage(obj URIObj, expiration time.Duration) (bool, error) { - if !IsStatusAllowed(obj.StatusCode) || !IsMethodAllowed(obj.Method) || expiration < 1 { - return false, nil +// IsValid - Verifies the validity of a cacheable object. +func (c CacheObj) IsValid() (bool, error) { + if !c.IsStatusAllowed() || slice.LenSliceBytes(c.CurrentObj.Content) == 0 { + return false, fmt.Errorf( + "not allowed. status %d - content length %d", + c.CurrentObj.StatusCode, + slice.LenSliceBytes(c.CurrentObj.Content), + ) } - targetURL := obj.URL - targetURL.Host = obj.Host + return true, nil +} - meta, err := GetVary(obj.ResponseHeaders) +func (c CacheObj) handleMetadata(targetURL url.URL, expiration time.Duration) ([]string, error) { + meta, err := GetVary(c.CurrentObj.ResponseHeaders) if err != nil { - return false, err + return []string{}, err } - _, err = StoreMetadata(obj.Method, targetURL, meta, expiration) + _, err = StoreMetadata(c.CurrentObj.Method, targetURL, meta, expiration) + if err != nil { + return []string{}, err + } + + return meta, nil +} + +// StoreFullPage - Stores the whole page response in cache. +func (c CacheObj) StoreFullPage(expiration time.Duration) (bool, error) { + if !c.IsStatusAllowed() || !c.IsMethodAllowed() || expiration < 1 { + return false, nil + } + + targetURL := c.CurrentObj.URL + targetURL.Host = c.CurrentObj.Host + + meta, err := c.handleMetadata(targetURL, expiration) if err != nil { return false, err } - encoded, err := engine.GetConn(targetURL.Host).Encode(obj) + encoded, err := engine.GetConn(targetURL.Host).Encode(c.CurrentObj) if err != nil { return false, err } - key := StorageKey(obj.Method, targetURL, meta, obj.RequestHeaders) + key := StorageKey(c.CurrentObj.Method, targetURL, meta, c.CurrentObj.RequestHeaders) return engine.GetConn(targetURL.Host).Set(key, encoded, expiration) } // RetrieveFullPage - Retrieves the whole page response from cache. -func RetrieveFullPage(method string, url url.URL, reqHeaders http.Header) (URIObj, error) { +func (c *CacheObj) RetrieveFullPage(method string, url url.URL, reqHeaders http.Header) error { obj := &URIObj{} meta, err := FetchMetadata(method, url) if err != nil { - return *obj, fmt.Errorf("cannot fetch metadata: %s", err) + return fmt.Errorf("cannot fetch metadata: %s", err) } key := StorageKey(method, url, meta, reqHeaders) @@ -87,20 +116,22 @@ func RetrieveFullPage(method string, url url.URL, reqHeaders http.Header) (URIOb encoded, err := engine.GetConn(url.Host).Get(key) if err != nil { - return *obj, fmt.Errorf("cannot get key: %s", err) + return fmt.Errorf("cannot get key: %s", err) } err = engine.GetConn(url.Host).Decode(encoded, obj) if err != nil { - return *obj, fmt.Errorf("cannot decode: %s", err) + return fmt.Errorf("cannot decode: %s", err) } - return *obj, nil + c.CurrentObj = *obj + + return nil } // PurgeFullPage - Deletes the whole page response from cache. -func PurgeFullPage(method string, url url.URL) (bool, error) { - err := DeleteMetadata(method, url) +func (c CacheObj) PurgeFullPage(method string, url url.URL) (bool, error) { + err := PurgeMetadata(url) if err != nil { return false, err } @@ -144,6 +175,14 @@ func FetchMetadata(method string, url url.URL) ([]string, error) { return engine.GetConn(url.Host).List(key) } +// PurgeMetadata - Purges the cache metadata for the requested URL. +func PurgeMetadata(url url.URL) error { + keyPattern := "META" + utils.StringSeparatorOne + "*" + utils.StringSeparatorOne + url.String() + + _, err := engine.GetConn(url.Host).DelWildcard(keyPattern) + return err +} + // DeleteMetadata - Removes the cache metadata for the requested URL. func DeleteMetadata(method string, url url.URL) error { key := "META" + utils.StringSeparatorOne + method + utils.StringSeparatorOne + url.String() diff --git a/cache/engine/client/client.go b/cache/engine/client/client.go index 0218f0e3..cf70b910 100644 --- a/cache/engine/client/client.go +++ b/cache/engine/client/client.go @@ -14,7 +14,9 @@ import ( "time" "github.com/fabiocicerchia/go-proxy-cache/config" - "github.com/fabiocicerchia/go-proxy-cache/utils" + "github.com/fabiocicerchia/go-proxy-cache/utils/base64" + circuitbreaker "github.com/fabiocicerchia/go-proxy-cache/utils/circuit-breaker" + "github.com/fabiocicerchia/go-proxy-cache/utils/msgpack" "github.com/go-redis/redis/v8" ) @@ -47,7 +49,7 @@ func (rdb *RedisClient) Close() error { // PurgeAll - Purges all the existing keys on a DB. func (rdb *RedisClient) PurgeAll() (bool, error) { - _, err := config.CB(rdb.Name).Execute(func() (interface{}, error) { + _, err := circuitbreaker.CB(rdb.Name).Execute(func() (interface{}, error) { err := rdb.Client.FlushDB(ctx).Err() return nil, err }) @@ -61,7 +63,7 @@ func (rdb *RedisClient) PurgeAll() (bool, error) { // Ping - Tests the connection. func (rdb *RedisClient) Ping() bool { - _, err := config.CB(rdb.Name).Execute(func() (interface{}, error) { + _, err := circuitbreaker.CB(rdb.Name).Execute(func() (interface{}, error) { err := rdb.Client.Ping(ctx).Err() return nil, err }) @@ -71,7 +73,7 @@ func (rdb *RedisClient) Ping() bool { // Set - Sets a key, with certain value, with TTL for expiring. func (rdb *RedisClient) Set(key string, value string, expiration time.Duration) (bool, error) { - _, err := config.CB(rdb.Name).Execute(func() (interface{}, error) { + _, err := circuitbreaker.CB(rdb.Name).Execute(func() (interface{}, error) { err := rdb.Client.Set(ctx, key, value, expiration).Err() return nil, err }) @@ -85,7 +87,7 @@ func (rdb *RedisClient) Set(key string, value string, expiration time.Duration) // Get - Gets a key. func (rdb *RedisClient) Get(key string) (string, error) { - value, err := config.CB(rdb.Name).Execute(func() (interface{}, error) { + value, err := circuitbreaker.CB(rdb.Name).Execute(func() (interface{}, error) { value, err := rdb.Client.Get(ctx, key).Result() if value == "" && err != nil && err.Error() == "redis: nil" { return "", nil @@ -103,7 +105,7 @@ func (rdb *RedisClient) Get(key string) (string, error) { // Del - Removes a key. func (rdb *RedisClient) Del(key string) error { - _, err := config.CB(rdb.Name).Execute(func() (interface{}, error) { + _, err := circuitbreaker.CB(rdb.Name).Execute(func() (interface{}, error) { err := rdb.Client.Del(ctx, key).Err() return nil, err }) @@ -113,7 +115,7 @@ func (rdb *RedisClient) Del(key string) error { // DelWildcard - Removes the matching keys based on a pattern. func (rdb *RedisClient) DelWildcard(key string) (int, error) { - k, err := config.CB(rdb.Name).Execute(func() (interface{}, error) { + k, err := circuitbreaker.CB(rdb.Name).Execute(func() (interface{}, error) { keys, err := rdb.Client.Keys(ctx, key).Result() return keys, err }) @@ -125,7 +127,7 @@ func (rdb *RedisClient) DelWildcard(key string) (int, error) { return l, nil } - _, errDel := config.CB(rdb.Name).Execute(func() (interface{}, error) { + _, errDel := circuitbreaker.CB(rdb.Name).Execute(func() (interface{}, error) { err := rdb.Client.Del(ctx, keys...).Err() return nil, err }) @@ -135,7 +137,7 @@ func (rdb *RedisClient) DelWildcard(key string) (int, error) { // List - Returns the values in a list. func (rdb *RedisClient) List(key string) ([]string, error) { - value, err := config.CB(rdb.Name).Execute(func() (interface{}, error) { + value, err := circuitbreaker.CB(rdb.Name).Execute(func() (interface{}, error) { value, err := rdb.Client.LRange(ctx, key, 0, -1).Result() return value, err }) @@ -149,7 +151,7 @@ func (rdb *RedisClient) List(key string) ([]string, error) { // Push - Append values to a list. func (rdb *RedisClient) Push(key string, values []string) error { - _, err := config.CB(rdb.Name).Execute(func() (interface{}, error) { + _, err := circuitbreaker.CB(rdb.Name).Execute(func() (interface{}, error) { err := rdb.Client.RPush(ctx, key, values).Err() return nil, err }) @@ -159,7 +161,7 @@ func (rdb *RedisClient) Push(key string, values []string) error { // Expire - Sets a TTL on a key. func (rdb *RedisClient) Expire(key string, expiration time.Duration) error { - _, err := config.CB(rdb.Name).Execute(func() (interface{}, error) { + _, err := circuitbreaker.CB(rdb.Name).Execute(func() (interface{}, error) { err := rdb.Client.Expire(ctx, key, expiration).Err() return nil, err }) @@ -169,23 +171,23 @@ func (rdb *RedisClient) Expire(key string, expiration time.Duration) error { // Encode - Encodes an object with msgpack. func (rdb *RedisClient) Encode(obj interface{}) (string, error) { - value, err := utils.MsgpackEncode(obj) + value, err := msgpack.Encode(obj) if err != nil { return "", err } - encoded := utils.Base64Encode(value) + encoded := base64.Encode(value) return encoded, nil } // Decode - Decodes an object with msgpack. func (rdb *RedisClient) Decode(encoded string, obj interface{}) error { - decoded, err := utils.Base64Decode(encoded) + decoded, err := base64.Decode(encoded) if err != nil { return err } - err = utils.MsgpackDecode(decoded, obj) + err = msgpack.Decode(decoded, obj) return err } diff --git a/cache/engine/client/client_test.go b/cache/engine/client/client_test.go index 0dda5596..f8f51ee4 100644 --- a/cache/engine/client/client_test.go +++ b/cache/engine/client/client_test.go @@ -20,6 +20,7 @@ import ( "github.com/fabiocicerchia/go-proxy-cache/cache/engine/client" "github.com/fabiocicerchia/go-proxy-cache/config" "github.com/fabiocicerchia/go-proxy-cache/utils" + circuit_breaker "github.com/fabiocicerchia/go-proxy-cache/utils/circuit-breaker" "github.com/stretchr/testify/assert" ) @@ -33,7 +34,7 @@ func TestCircuitBreakerWithPingTimeout(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: 0, // doesn't clears counts @@ -41,27 +42,27 @@ func TestCircuitBreakerWithPingTimeout(t *testing.T) { }, } - config.InitCircuitBreaker("testing", config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker("testing", config.Config.CircuitBreaker) rdb := client.Connect("testing", config.Config.Cache) - assert.Equal(t, "closed", config.CB("testing").State().String()) + assert.Equal(t, "closed", circuit_breaker.CB("testing").State().String()) val := rdb.Ping() assert.True(t, val) - assert.Equal(t, "closed", config.CB("testing").State().String()) + assert.Equal(t, "closed", circuit_breaker.CB("testing").State().String()) _ = rdb.Close() val = rdb.Ping() assert.False(t, val) - assert.Equal(t, "half-open", config.CB("testing").State().String()) + assert.Equal(t, "half-open", circuit_breaker.CB("testing").State().String()) rdb = client.Connect("testing", config.Config.Cache) val = rdb.Ping() assert.True(t, val) - assert.Equal(t, "closed", config.CB("testing").State().String()) + assert.Equal(t, "closed", circuit_breaker.CB("testing").State().String()) } func TestClose(t *testing.T) { @@ -74,7 +75,7 @@ func TestClose(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: 0, // doesn't clears counts @@ -82,7 +83,7 @@ func TestClose(t *testing.T) { }, } - config.InitCircuitBreaker("testing", config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker("testing", config.Config.CircuitBreaker) rdb := client.Connect("testing", config.Config.Cache) @@ -103,7 +104,7 @@ func TestSetGet(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: 0, // doesn't clears counts @@ -111,7 +112,7 @@ func TestSetGet(t *testing.T) { }, } - config.InitCircuitBreaker("testing", config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker("testing", config.Config.CircuitBreaker) rdb := client.Connect("testing", config.Config.Cache) @@ -134,7 +135,7 @@ func TestSetGetWithExpiration(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: 0, // doesn't clears counts @@ -142,7 +143,7 @@ func TestSetGetWithExpiration(t *testing.T) { }, } - config.InitCircuitBreaker("testing", config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker("testing", config.Config.CircuitBreaker) rdb := client.Connect("testing", config.Config.Cache) @@ -167,7 +168,7 @@ func TestDel(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: 0, // doesn't clears counts @@ -175,7 +176,7 @@ func TestDel(t *testing.T) { }, } - config.InitCircuitBreaker("testing", config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker("testing", config.Config.CircuitBreaker) rdb := client.Connect("testing", config.Config.Cache) @@ -205,7 +206,7 @@ func TestExpire(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: 0, // doesn't clears counts @@ -213,7 +214,7 @@ func TestExpire(t *testing.T) { }, } - config.InitCircuitBreaker("testing", config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker("testing", config.Config.CircuitBreaker) rdb := client.Connect("testing", config.Config.Cache) @@ -241,7 +242,7 @@ func TestPushList(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: 0, // doesn't clears counts @@ -249,7 +250,7 @@ func TestPushList(t *testing.T) { }, } - config.InitCircuitBreaker("testing", config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker("testing", config.Config.CircuitBreaker) rdb := client.Connect("testing", config.Config.Cache) @@ -271,7 +272,7 @@ func TestDelWildcard(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: 0, // doesn't clears counts @@ -279,7 +280,7 @@ func TestDelWildcard(t *testing.T) { }, } - config.InitCircuitBreaker("testing", config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker("testing", config.Config.CircuitBreaker) rdb := client.Connect("testing", config.Config.Cache) @@ -328,7 +329,7 @@ func TestPurgeAll(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: 0, // doesn't clears counts @@ -336,7 +337,7 @@ func TestPurgeAll(t *testing.T) { }, } - config.InitCircuitBreaker("testing", config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker("testing", config.Config.CircuitBreaker) rdb := client.Connect("testing", config.Config.Cache) @@ -385,7 +386,7 @@ func TestEncodeDecode(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: 0, // doesn't clears counts @@ -393,7 +394,7 @@ func TestEncodeDecode(t *testing.T) { }, } - config.InitCircuitBreaker("testing", config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker("testing", config.Config.CircuitBreaker) rdb := client.Connect("testing", config.Config.Cache) diff --git a/config/config.go b/config/config.go index 18c7d140..6e83f822 100644 --- a/config/config.go +++ b/config/config.go @@ -12,6 +12,7 @@ package config import ( "crypto/tls" "io/ioutil" + "os" "path/filepath" "strings" "time" @@ -20,19 +21,88 @@ import ( "gopkg.in/yaml.v2" "github.com/fabiocicerchia/go-proxy-cache/utils" - "github.com/sony/gobreaker" + circuitbreaker "github.com/fabiocicerchia/go-proxy-cache/utils/circuit-breaker" + "github.com/fabiocicerchia/go-proxy-cache/utils/convert" + "github.com/fabiocicerchia/go-proxy-cache/utils/slice" ) // Config - Holds the server configuration -var Config Configuration -var cb map[string]*gobreaker.CircuitBreaker +var Config Configuration = Configuration{ + Server: Server{ + Port: Port{ + HTTP: "80", + HTTPS: "443", + }, + TLS: TLS{ + Auto: false, + Email: "", + CertFile: "", + KeyFile: "", + Override: &tls.Config{ + // TODO: handle this + // Use modern tls mode https://wiki.mozilla.org/Security/Server_Side_TLS#Modern_compatibility + // NextProtos: []string{"h2", "http/1.1"}, + // Only use curves which have assembly implementations + // https://github.com/golang/go/tree/master/src/crypto/elliptic + CurvePreferences: []tls.CurveID{ + tls.CurveP256, + }, + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS13, + CipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + // needed by HTTP/2 + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + }, + }, + }, + Timeout: Timeout{ + Read: 5 * time.Second, + ReadHeader: 2 * time.Second, + Write: 5 * time.Second, + Idle: 20 * time.Second, + Handler: 5 * time.Second, + }, + Forwarding: Forward{ + HTTP2HTTPS: false, + InsecureBridge: false, + RedirectStatusCode: 301, + }, + GZip: false, + Healthcheck: true, + }, + Cache: Cache{ + Port: "6379", + DB: 0, + TTL: 0, + AllowedStatuses: []int{200, 301, 302}, + AllowedMethods: []string{"HEAD", "GET"}, + }, + CircuitBreaker: circuitbreaker.CircuitBreaker{ + Threshold: 2, // after 2nd request, if meet FailureRate goes open. + FailureRate: 0.5, // 1 out of 2 fails, or more + Interval: 0, // doesn't clears counts + Timeout: 60 * time.Second, // clears state after 60s + MaxRequests: 1, + }, + Log: Log{ + TimeFormat: "2006/01/02 15:04:05", + Format: `$host - $remote_addr - $remote_user $protocol $request_method "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $cached_status`, + }, +} + var allowedSchemes = map[string]string{"HTTP": "http", "HTTPS": "https"} // Configuration - Defines the server configuration type Configuration struct { Server Server Cache Cache - CircuitBreaker CircuitBreaker + CircuitBreaker circuitbreaker.CircuitBreaker Domains Domains Log Log } @@ -96,91 +166,12 @@ type Cache struct { AllowedMethods []string } -// CircuitBreaker - Settings for redis circuit breaker. -type CircuitBreaker struct { - Threshold uint32 - FailureRate float64 - Interval time.Duration - Timeout time.Duration - MaxRequests uint32 -} - +// Log - Defines the config for the logs type Log struct { TimeFormat string Format string } -func getDefaultConfig() Configuration { - return Configuration{ - Server: Server{ - Port: Port{ - HTTP: "80", - HTTPS: "443", - }, - TLS: TLS{ - Auto: false, - Email: "", - CertFile: "", - KeyFile: "", - Override: &tls.Config{ - // TODO: handle this - // Use modern tls mode https://wiki.mozilla.org/Security/Server_Side_TLS#Modern_compatibility - // NextProtos: []string{"h2", "http/1.1"}, - // Only use curves which have assembly implementations - // https://github.com/golang/go/tree/master/src/crypto/elliptic - CurvePreferences: []tls.CurveID{ - tls.CurveP256, - }, - MinVersion: tls.VersionTLS12, - MaxVersion: tls.VersionTLS13, - CipherSuites: []uint16{ - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, - // needed by HTTP/2 - tls.TLS_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - }, - }, - }, - Timeout: Timeout{ - Read: 5 * time.Second, - ReadHeader: 2 * time.Second, - Write: 5 * time.Second, - Idle: 20 * time.Second, - Handler: 5 * time.Second, - }, - Forwarding: Forward{ - HTTP2HTTPS: false, - InsecureBridge: false, - RedirectStatusCode: 301, - }, - GZip: false, - Healthcheck: true, - }, - Cache: Cache{ - Port: "6379", - DB: 0, - TTL: 0, - AllowedStatuses: []int{200, 301, 302}, - AllowedMethods: []string{"HEAD", "GET"}, - }, - CircuitBreaker: CircuitBreaker{ - Threshold: 2, // after 2nd request, if meet FailureRate goes open. - FailureRate: 0.5, // 1 out of 2 fails, or more - Interval: 0, // doesn't clears counts - Timeout: 60 * time.Second, // clears state after 60s - MaxRequests: 1, - }, - Log: Log{ - TimeFormat: "2006/01/02 15:04:05", - Format: `$host - $remote_addr - $remote_user $protocol $request_method "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $cached_status`, - }, - } -} - func normalizeScheme(scheme string) string { schemeUpper := strings.ToUpper(scheme) if val, ok := allowedSchemes[schemeUpper]; ok { @@ -191,48 +182,43 @@ func normalizeScheme(scheme string) string { } func getEnvConfig() Configuration { - return Configuration{ - Server: Server{ - Port: Port{ - HTTP: utils.GetEnv("SERVER_HTTP_PORT", ""), - HTTPS: utils.GetEnv("SERVER_HTTPS_PORT", ""), - }, - TLS: TLS{ - Auto: utils.GetEnv("TLS_AUTO_CERT", "") == "1", - Email: utils.GetEnv("TLS_EMAIL", ""), - CertFile: utils.GetEnv("TLS_CERT_FILE", ""), - KeyFile: utils.GetEnv("TLS_KEY_FILE", ""), - }, - Timeout: Timeout{ - Read: utils.ConvertToDuration(utils.GetEnv("TIMEOUT_READ", "")), - ReadHeader: utils.ConvertToDuration(utils.GetEnv("TIMEOUT_READ_HEADER", "")), - Write: utils.ConvertToDuration(utils.GetEnv("TIMEOUT_WRITE", "")), - Idle: utils.ConvertToDuration(utils.GetEnv("TIMEOUT_IDLE", "")), - Handler: utils.ConvertToDuration(utils.GetEnv("TIMEOUT_HANDLER", "")), - }, - Forwarding: Forward{ - Host: utils.GetEnv("FORWARD_HOST", ""), - Port: utils.GetEnv("FORWARD_PORT", ""), - Scheme: normalizeScheme(utils.GetEnv("FORWARD_SCHEME", "")), - Endpoints: strings.Split(utils.GetEnv("LB_ENDPOINT_LIST", ""), ","), - HTTP2HTTPS: utils.GetEnv("HTTP2HTTPS", "") == "1", - RedirectStatusCode: utils.ConvertToInt(utils.GetEnv("REDIRECT_STATUS_CODE", "")), - }, - GZip: utils.GetEnv("GZIP_ENABLED", "") == "1", - }, - Cache: Cache{ - Host: utils.GetEnv("REDIS_HOST", ""), - Port: utils.GetEnv("REDIS_PORT", ""), - Password: utils.GetEnv("REDIS_PASSWORD", ""), - DB: utils.ConvertToInt(utils.GetEnv("REDIS_DB", "")), - TTL: utils.ConvertToInt(utils.GetEnv("DEFAULT_TTL", "")), - AllowedStatuses: utils.ConvertToIntSlice(strings.Split(utils.GetEnv("CACHE_ALLOWED_STATUSES", ""), ",")), - AllowedMethods: strings.Split(utils.GetEnv("CACHE_ALLOWED_METHODS", ""), ","), - }, - } + EnvConfig := Configuration{} + + EnvConfig.Server.Port.HTTP = utils.GetEnv("SERVER_HTTP_PORT", "") + EnvConfig.Server.Port.HTTPS = utils.GetEnv("SERVER_HTTPS_PORT", "") + + EnvConfig.Server.TLS.Auto = utils.GetEnv("TLS_AUTO_CERT", "") == "1" + EnvConfig.Server.TLS.Email = utils.GetEnv("TLS_EMAIL", "") + EnvConfig.Server.TLS.CertFile = utils.GetEnv("TLS_CERT_FILE", "") + EnvConfig.Server.TLS.KeyFile = utils.GetEnv("TLS_KEY_FILE", "") + + EnvConfig.Server.Timeout.Read = convert.ToDuration(utils.GetEnv("TIMEOUT_READ", "")) + EnvConfig.Server.Timeout.ReadHeader = convert.ToDuration(utils.GetEnv("TIMEOUT_READ_HEADER", "")) + EnvConfig.Server.Timeout.Write = convert.ToDuration(utils.GetEnv("TIMEOUT_WRITE", "")) + EnvConfig.Server.Timeout.Idle = convert.ToDuration(utils.GetEnv("TIMEOUT_IDLE", "")) + EnvConfig.Server.Timeout.Handler = convert.ToDuration(utils.GetEnv("TIMEOUT_HANDLER", "")) + + EnvConfig.Server.Forwarding.Host = utils.GetEnv("FORWARD_HOST", "") + EnvConfig.Server.Forwarding.Port = utils.GetEnv("FORWARD_PORT", "") + EnvConfig.Server.Forwarding.Scheme = normalizeScheme(utils.GetEnv("FORWARD_SCHEME", "")) + EnvConfig.Server.Forwarding.Endpoints = strings.Split(utils.GetEnv("LB_ENDPOINT_LIST", ""), ",") + EnvConfig.Server.Forwarding.HTTP2HTTPS = utils.GetEnv("HTTP2HTTPS", "") == "1" + EnvConfig.Server.Forwarding.RedirectStatusCode = convert.ToInt(utils.GetEnv("REDIRECT_STATUS_CODE", "")) + + EnvConfig.Server.GZip = utils.GetEnv("GZIP_ENABLED", "") == "1" + + EnvConfig.Cache.Host = utils.GetEnv("REDIS_HOST", "") + EnvConfig.Cache.Port = utils.GetEnv("REDIS_PORT", "") + EnvConfig.Cache.Password = utils.GetEnv("REDIS_PASSWORD", "") + EnvConfig.Cache.DB = convert.ToInt(utils.GetEnv("REDIS_DB", "")) + EnvConfig.Cache.TTL = convert.ToInt(utils.GetEnv("DEFAULT_TTL", "")) + EnvConfig.Cache.AllowedStatuses = convert.ToIntSlice(strings.Split(utils.GetEnv("CACHE_ALLOWED_STATUSES", ""), ",")) + EnvConfig.Cache.AllowedMethods = strings.Split(utils.GetEnv("CACHE_ALLOWED_METHODS", ""), ",") + + return EnvConfig } -func getYamlConfig(file string, strict bool) (Configuration, error) { +func getYamlConfig(file string) (Configuration, error) { YamlConfig := Configuration{} data, err := ioutil.ReadFile(filepath.Clean(file)) @@ -240,11 +226,7 @@ func getYamlConfig(file string, strict bool) (Configuration, error) { return YamlConfig, err } - if strict { - err = yaml.UnmarshalStrict([]byte(data), &YamlConfig) - } else { - err = yaml.Unmarshal([]byte(data), &YamlConfig) - } + err = yaml.UnmarshalStrict([]byte(data), &YamlConfig) if err != nil { return YamlConfig, err @@ -258,14 +240,16 @@ func getYamlConfig(file string, strict bool) (Configuration, error) { // InitConfigFromFileOrEnv - Init the configuration in sequence: from a YAML file, from environment variables, // then defaults. func InitConfigFromFileOrEnv(file string) { - Config = Configuration{} - Config = CopyOverWith(Config, getDefaultConfig()) Config = CopyOverWith(Config, getEnvConfig()) - YamlConfig, err := getYamlConfig(file, false) - if err == nil { + + var YamlConfig Configuration + _, err := os.Stat(file) + if !os.IsNotExist(err) { + YamlConfig, err = getYamlConfig(file) + if err != nil { + log.Fatalf("Cannot unmarshal YAML: %s\n", err) + } Config = CopyOverWith(Config, YamlConfig) - } else { - log.Warnf("Cannot unmarshal YAML: %s\n", err) } // allow only the config file to specify overrides per domain @@ -286,7 +270,7 @@ func InitConfigFromFileOrEnv(file string) { // Validate - Validate a YAML config file is syntactically valid. func Validate(file string) (bool, error) { - _, err := getYamlConfig(file, true) + _, err := getYamlConfig(file) return err != nil, err } @@ -295,64 +279,44 @@ func CopyOverWith(base Configuration, overrides Configuration) Configuration { newConf := base // --- SERVER - - serverN := newConf.Server - serverO := overrides.Server - serverN.Port.HTTP = utils.Coalesce(serverO.Port.HTTP, serverN.Port.HTTP, serverO.Port.HTTP == "").(string) - serverN.Port.HTTPS = utils.Coalesce(serverO.Port.HTTPS, serverN.Port.HTTPS, serverO.Port.HTTPS == "").(string) - serverN.GZip = utils.Coalesce(serverO.GZip, serverN.GZip, !serverO.GZip).(bool) - newConf.Server = serverN + newConf.Server.Port.HTTP = utils.Coalesce(overrides.Server.Port.HTTP, newConf.Server.Port.HTTP, overrides.Server.Port.HTTP == "").(string) + newConf.Server.Port.HTTPS = utils.Coalesce(overrides.Server.Port.HTTPS, newConf.Server.Port.HTTPS, overrides.Server.Port.HTTPS == "").(string) + newConf.Server.GZip = utils.Coalesce(overrides.Server.GZip, newConf.Server.GZip, !overrides.Server.GZip).(bool) // --- TLS - - tlsN := newConf.Server.TLS - tlsO := overrides.Server.TLS - tlsN.Auto = utils.Coalesce(tlsO.Auto, tlsN.Auto, !tlsO.Auto).(bool) - tlsN.Email = utils.Coalesce(tlsO.Email, tlsN.Email, tlsO.Email == "").(string) - tlsN.CertFile = utils.Coalesce(tlsO.CertFile, tlsN.CertFile, tlsO.CertFile == "").(string) - tlsN.KeyFile = utils.Coalesce(tlsO.KeyFile, tlsN.KeyFile, tlsO.KeyFile == "").(string) - tlsN.Override = utils.Coalesce(tlsO.Override, tlsN.Override, tlsO.Override == nil).(*tls.Config) - newConf.Server.TLS = tlsN + newConf.Server.TLS.Auto = utils.Coalesce(overrides.Server.TLS.Auto, newConf.Server.TLS.Auto, !overrides.Server.TLS.Auto).(bool) + newConf.Server.TLS.Email = utils.Coalesce(overrides.Server.TLS.Email, newConf.Server.TLS.Email, overrides.Server.TLS.Email == "").(string) + newConf.Server.TLS.CertFile = utils.Coalesce(overrides.Server.TLS.CertFile, newConf.Server.TLS.CertFile, overrides.Server.TLS.CertFile == "").(string) + newConf.Server.TLS.KeyFile = utils.Coalesce(overrides.Server.TLS.KeyFile, newConf.Server.TLS.KeyFile, overrides.Server.TLS.KeyFile == "").(string) + newConf.Server.TLS.Override = utils.Coalesce(overrides.Server.TLS.Override, newConf.Server.TLS.Override, overrides.Server.TLS.Override == nil).(*tls.Config) // --- Timeout - - timeoutN := newConf.Server.Timeout - timeoutO := overrides.Server.Timeout - timeoutN.Read = utils.Coalesce(timeoutO.Read, timeoutN.Read, timeoutO.Read == 0).(time.Duration) - timeoutN.ReadHeader = utils.Coalesce(timeoutO.ReadHeader, timeoutN.ReadHeader, timeoutO.ReadHeader == 0).(time.Duration) - timeoutN.Write = utils.Coalesce(timeoutO.Write, timeoutN.Write, timeoutO.Write == 0).(time.Duration) - timeoutN.Idle = utils.Coalesce(timeoutO.Idle, timeoutN.Idle, timeoutO.Idle == 0).(time.Duration) - timeoutN.Handler = utils.Coalesce(timeoutO.Handler, timeoutN.Handler, timeoutO.Handler == 0).(time.Duration) - newConf.Server.Timeout = timeoutN + newConf.Server.Timeout.Read = utils.Coalesce(overrides.Server.Timeout.Read, newConf.Server.Timeout.Read, overrides.Server.Timeout.Read == 0).(time.Duration) + newConf.Server.Timeout.ReadHeader = utils.Coalesce(overrides.Server.Timeout.ReadHeader, newConf.Server.Timeout.ReadHeader, overrides.Server.Timeout.ReadHeader == 0).(time.Duration) + newConf.Server.Timeout.Write = utils.Coalesce(overrides.Server.Timeout.Write, newConf.Server.Timeout.Write, overrides.Server.Timeout.Write == 0).(time.Duration) + newConf.Server.Timeout.Idle = utils.Coalesce(overrides.Server.Timeout.Idle, newConf.Server.Timeout.Idle, overrides.Server.Timeout.Idle == 0).(time.Duration) + newConf.Server.Timeout.Handler = utils.Coalesce(overrides.Server.Timeout.Handler, newConf.Server.Timeout.Handler, overrides.Server.Timeout.Handler == 0).(time.Duration) // --- Forwarding - - forwardingN := newConf.Server.Forwarding - forwardingO := overrides.Server.Forwarding - forwardingN.Host = utils.Coalesce(forwardingO.Host, forwardingN.Host, forwardingO.Host == "").(string) - forwardingN.Port = utils.Coalesce(forwardingO.Port, forwardingN.Port, forwardingO.Port == "").(string) - forwardingN.Scheme = utils.Coalesce(forwardingO.Scheme, forwardingN.Scheme, forwardingO.Scheme == "").(string) - forwardingN.Endpoints = utils.Coalesce(forwardingO.Endpoints, forwardingN.Endpoints, len(forwardingO.Endpoints) == 0).([]string) - forwardingN.HTTP2HTTPS = utils.Coalesce(forwardingO.HTTP2HTTPS, forwardingN.HTTP2HTTPS, !forwardingO.HTTP2HTTPS).(bool) - forwardingN.InsecureBridge = utils.Coalesce(forwardingO.InsecureBridge, forwardingN.InsecureBridge, !forwardingO.InsecureBridge).(bool) - forwardingN.RedirectStatusCode = utils.Coalesce(forwardingO.RedirectStatusCode, forwardingN.RedirectStatusCode, forwardingO.RedirectStatusCode == 0).(int) - newConf.Server.Forwarding = forwardingN + newConf.Server.Forwarding.Host = utils.Coalesce(overrides.Server.Forwarding.Host, newConf.Server.Forwarding.Host, overrides.Server.Forwarding.Host == "").(string) + newConf.Server.Forwarding.Port = utils.Coalesce(overrides.Server.Forwarding.Port, newConf.Server.Forwarding.Port, overrides.Server.Forwarding.Port == "").(string) + newConf.Server.Forwarding.Scheme = utils.Coalesce(overrides.Server.Forwarding.Scheme, newConf.Server.Forwarding.Scheme, overrides.Server.Forwarding.Scheme == "").(string) + newConf.Server.Forwarding.Endpoints = utils.Coalesce(overrides.Server.Forwarding.Endpoints, newConf.Server.Forwarding.Endpoints, len(overrides.Server.Forwarding.Endpoints) == 0).([]string) + newConf.Server.Forwarding.HTTP2HTTPS = utils.Coalesce(overrides.Server.Forwarding.HTTP2HTTPS, newConf.Server.Forwarding.HTTP2HTTPS, !overrides.Server.Forwarding.HTTP2HTTPS).(bool) + newConf.Server.Forwarding.InsecureBridge = utils.Coalesce(overrides.Server.Forwarding.InsecureBridge, newConf.Server.Forwarding.InsecureBridge, !overrides.Server.Forwarding.InsecureBridge).(bool) + newConf.Server.Forwarding.RedirectStatusCode = utils.Coalesce(overrides.Server.Forwarding.RedirectStatusCode, newConf.Server.Forwarding.RedirectStatusCode, overrides.Server.Forwarding.RedirectStatusCode == 0).(int) // --- Cache + newConf.Cache.Host = utils.Coalesce(overrides.Cache.Host, newConf.Cache.Host, overrides.Cache.Host == "").(string) + newConf.Cache.Port = utils.Coalesce(overrides.Cache.Port, newConf.Cache.Port, overrides.Cache.Port == "").(string) + newConf.Cache.Password = utils.Coalesce(overrides.Cache.Password, newConf.Cache.Password, overrides.Cache.Password == "").(string) + newConf.Cache.DB = utils.Coalesce(overrides.Cache.DB, newConf.Cache.DB, overrides.Cache.DB == 0).(int) + newConf.Cache.TTL = utils.Coalesce(overrides.Cache.TTL, newConf.Cache.TTL, overrides.Cache.TTL == 0).(int) + newConf.Cache.AllowedStatuses = utils.Coalesce(overrides.Cache.AllowedStatuses, newConf.Cache.AllowedStatuses, len(overrides.Cache.AllowedStatuses) == 0).([]int) + newConf.Cache.AllowedMethods = utils.Coalesce(overrides.Cache.AllowedMethods, newConf.Cache.AllowedMethods, len(overrides.Cache.AllowedMethods) == 0).([]string) - cacheN := newConf.Cache - cacheO := overrides.Cache - cacheN.Host = utils.Coalesce(cacheO.Host, newConf.Cache.Host, cacheO.Host == "").(string) - cacheN.Port = utils.Coalesce(cacheO.Port, newConf.Cache.Port, cacheO.Port == "").(string) - cacheN.Password = utils.Coalesce(cacheO.Password, newConf.Cache.Password, cacheO.Password == "").(string) - cacheN.DB = utils.Coalesce(cacheO.DB, newConf.Cache.DB, cacheO.DB == 0).(int) - cacheN.TTL = utils.Coalesce(cacheO.TTL, newConf.Cache.TTL, cacheO.TTL == 0).(int) - cacheN.AllowedStatuses = utils.Coalesce(cacheO.AllowedStatuses, newConf.Cache.AllowedStatuses, len(cacheO.AllowedStatuses) == 0).([]int) - cacheN.AllowedMethods = utils.Coalesce(cacheO.AllowedMethods, newConf.Cache.AllowedMethods, len(cacheO.AllowedMethods) == 0).([]string) - - cacheN.AllowedMethods = append(cacheN.AllowedMethods, "HEAD", "GET") - cacheN.AllowedMethods = utils.Unique(cacheN.AllowedMethods) - newConf.Cache = cacheN + newConf.Cache.AllowedMethods = append(newConf.Cache.AllowedMethods, "HEAD", "GET") + newConf.Cache.AllowedMethods = slice.Unique(newConf.Cache.AllowedMethods) return newConf } @@ -371,6 +335,7 @@ func Print() { // GetDomains - Returns a list of domains. func GetDomains() []string { + // TODO: What if there's no domains only main config?! domains := make([]string, 0, len(Config.Domains)) for _, v := range Config.Domains { domains = append(domains, v.Server.Forwarding.Host) @@ -396,36 +361,3 @@ func DomainConf(domain string) *Configuration { return nil } - -// InitCircuitBreaker - Initialise the Circuit Breaker. -func InitCircuitBreaker(name string, config CircuitBreaker) { - st := gobreaker.Settings{ - Name: name, - MaxRequests: config.MaxRequests, - Interval: config.Interval, - Timeout: config.Timeout, - ReadyToTrip: func(counts gobreaker.Counts) bool { - failureRatio := float64(counts.TotalFailures) / float64(counts.Requests) - return counts.Requests >= config.Threshold && failureRatio >= config.FailureRate - }, - OnStateChange: func(name string, from gobreaker.State, to gobreaker.State) { - log.Warnf("Circuit Breaker - Changed from %s to %s", from.String(), to.String()) - }, - } - - if cb == nil { - cb = make(map[string]*gobreaker.CircuitBreaker) - } - - cb[name] = gobreaker.NewCircuitBreaker(st) -} - -// CB - Returns instance of gobreaker.CircuitBreaker. -func CB(name string) *gobreaker.CircuitBreaker { - if val, ok := cb[name]; ok { - return val - } - - log.Warnf("Missing circuit breaker for %s", name) - return nil -} diff --git a/go.mod b/go.mod index 8ec55b5c..4c91ef67 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,13 @@ go 1.14 require ( github.com/NYTimes/gziphandler v1.1.1 github.com/go-redis/redis/v8 v8.3.3 + github.com/kr/pretty v0.1.0 // indirect github.com/sirupsen/logrus v1.3.0 github.com/sony/gobreaker v0.4.1 github.com/stretchr/testify v1.6.1 github.com/ugorji/go/codec v1.2.0 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v2 v2.3.0 - honnef.co/go/tools v0.0.1-2020.1.6 // indirect ) diff --git a/go.sum b/go.sum index f0ca716a..9e65c73c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= @@ -28,9 +26,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -50,7 +46,6 @@ github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sony/gobreaker v0.4.1 h1:oMnRNZXX5j85zso6xCPRNPtmAycat+WcoKbklScLDgQ= @@ -65,25 +60,18 @@ github.com/ugorji/go v1.2.0 h1:6eXlzYLLwZwXroJx9NyqbYcbv/d93twiOzQLDewE6qM= github.com/ugorji/go v1.2.0/go.mod h1:1ny++pKMXhLWrwWV5Nf+CbOuZJhMoaFD+0GMFfd8fEc= github.com/ugorji/go/codec v1.2.0 h1:As6RccOIlbm9wHuWYMlB30dErcI+4WiKWsYsmPkyrUw= github.com/ugorji/go/codec v1.2.0/go.mod h1:dXvG35r7zTX6QImXOSFhGMmKtX+wJ7VTWzGvYQGIjBs= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/otel v0.13.0 h1:2isEnyzjjJZq6r2EKMsFj4TxiQiexsM04AVhwbR/oBA= go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -100,11 +88,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef h1:RHORRhs540cYZYrzgU2CPUyykkwZM78hGdzocOo9P8A= -golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -117,7 +100,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -126,5 +108,3 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.1-2020.1.6 h1:W18jzjh8mfPez+AwGLxmOImucz/IFjpNlrKVnaj2YVc= -honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY= diff --git a/main.go b/main.go index 50ce1b0b..18035264 100644 --- a/main.go +++ b/main.go @@ -94,6 +94,5 @@ func main() { initFlags() initLogs() - // start server - server.Start(configFile) + server.Run(configFile) } diff --git a/server/handler/handler.go b/server/handler/handler.go new file mode 100644 index 00000000..e880a005 --- /dev/null +++ b/server/handler/handler.go @@ -0,0 +1,103 @@ +package handler + +// __ +// .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. +// | _ | _ |______| _ | _| _ |_ _| | |______| __| _ | __| | -__| +// |___ |_____| | __|__| |_____|__.__|___ | |____|___._|____|__|__|_____| +// |_____| |__| |_____| +// +// Copyright (c) 2020 Fabio Cicerchia. https://fabiocicerchia.it. MIT License +// Repo: https://github.com/fabiocicerchia/go-proxy-cache + +import ( + "context" + "net" + "net/http" + "strconv" + + "github.com/fabiocicerchia/go-proxy-cache/cache" + "github.com/fabiocicerchia/go-proxy-cache/config" + "github.com/fabiocicerchia/go-proxy-cache/server/logger" + "github.com/fabiocicerchia/go-proxy-cache/server/response" + "github.com/fabiocicerchia/go-proxy-cache/server/storage" + log "github.com/sirupsen/logrus" +) + +// RequestCall - Main object containing request and response. +type RequestCall struct { + Response *response.LoggedResponseWriter + Request *http.Request +} + +// ConvertToRequestCallDTO - Generates a storage DTO containing request, response and cache settings. +func ConvertToRequestCallDTO(rc RequestCall) storage.RequestCallDTO { + return storage.RequestCallDTO{ + Response: *rc.Response, + Request: *rc.Request, + // TODO: convert to use domainConfigCache + CacheObj: cache.CacheObj{ + AllowedStatuses: config.Config.Cache.AllowedStatuses, + AllowedMethods: config.Config.Cache.AllowedMethods, + }, + } +} + +func getListeningPort(ctx context.Context) string { + localAddrContextKey := ctx.Value(http.LocalAddrContextKey) + listeningPort := "" + if localAddrContextKey != nil { + srvAddr := localAddrContextKey.(*net.TCPAddr) + listeningPort = strconv.Itoa(srvAddr.Port) + } + + return listeningPort +} + +// HandleRequest - Handles the entrypoint and directs the traffic to the right handler. +func HandleRequest(res http.ResponseWriter, req *http.Request) { + rc := RequestCall{ + Response: response.NewLoggedResponseWriter(res), + Request: req, + } + + listeningPort := getListeningPort(req.Context()) + + domainConfig := config.DomainConf(req.Host) + if domainConfig == nil || + (domainConfig.Server.Port.HTTP != listeningPort && + domainConfig.Server.Port.HTTPS != listeningPort) { + rc.Response.WriteHeader(http.StatusNotImplemented) + logger.LogRequest(*rc.Request, *rc.Response, false) + log.Errorf("Missing configuration in HandleRequest for %s (listening on :%s).", rc.Request.Host, listeningPort) + return + } + + if rc.GetScheme() == "http" && domainConfig.Server.Forwarding.HTTP2HTTPS { + rc.RedirectToHTTPS(domainConfig.Server.Forwarding.RedirectStatusCode) + return + } + + if rc.Request.Method == "PURGE" { + rc.HandlePurge(domainConfig) + return + } + + if req.Method == http.MethodConnect { + rc.Response.WriteHeader(http.StatusMethodNotAllowed) + } else { + rc.HandleRequestAndProxy(domainConfig) + } +} + +// GetScheme - Returns current request scheme +// For server requests the URL is parsed from the URI supplied on the +// Request-Line as stored in RequestURI. For most requests, fields other than +// Path and RawQuery will be empty. (See RFC 7230, Section 5.3) +// Ref: https://github.com/golang/go/issues/28940 +func (rc RequestCall) GetScheme() string { + if rc.Request.TLS != nil { + // TODO: COVERAGE + return "https" + } + return "http" +} diff --git a/server/handler/healthcheck.go b/server/handler/healthcheck.go index 1c64027c..ed085872 100644 --- a/server/handler/healthcheck.go +++ b/server/handler/healthcheck.go @@ -21,11 +21,11 @@ func HandleHealthcheck(res http.ResponseWriter, req *http.Request) { lwr := response.NewLoggedResponseWriter(res) lwr.WriteHeader(http.StatusOK) - _ = response.WriteBody(lwr, "HTTP OK\n") + _ = lwr.WriteBody("HTTP OK\n") if conn := engine.GetConn(req.Host); conn != nil && conn.Ping() { - _ = response.WriteBody(lwr, "REDIS OK\n") + _ = lwr.WriteBody("REDIS OK\n") } else { - _ = response.WriteBody(lwr, "REDIS KO\n") + _ = lwr.WriteBody("REDIS KO\n") } } diff --git a/server/handler/healthcheck_test.go b/server/handler/healthcheck_test.go index 48ea0946..b40a0e33 100644 --- a/server/handler/healthcheck_test.go +++ b/server/handler/healthcheck_test.go @@ -24,6 +24,7 @@ import ( "github.com/fabiocicerchia/go-proxy-cache/config" "github.com/fabiocicerchia/go-proxy-cache/server/handler" "github.com/fabiocicerchia/go-proxy-cache/utils" + circuit_breaker "github.com/fabiocicerchia/go-proxy-cache/utils/circuit-breaker" ) func TestHealthcheckWithoutRedis(t *testing.T) { @@ -36,7 +37,7 @@ func TestHealthcheckWithoutRedis(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: time.Duration(1), @@ -44,7 +45,7 @@ func TestHealthcheckWithoutRedis(t *testing.T) { }, } - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) engine.GetConn(config.Config.Server.Forwarding.Host).Close() @@ -75,7 +76,7 @@ func TestHealthcheckWithRedis(t *testing.T) { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: time.Duration(1), @@ -89,7 +90,7 @@ func TestHealthcheckWithRedis(t *testing.T) { rr := httptest.NewRecorder() h := http.HandlerFunc(handler.HandleHealthcheck) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) diff --git a/server/handler/http.go b/server/handler/http.go index 6fcc7a20..b56459d3 100644 --- a/server/handler/http.go +++ b/server/handler/http.go @@ -10,14 +10,11 @@ package handler // Repo: https://github.com/fabiocicerchia/go-proxy-cache import ( - "context" "crypto/tls" "fmt" - "net" "net/http" "net/http/httputil" "net/url" - "strconv" "strings" "github.com/fabiocicerchia/go-proxy-cache/config" @@ -30,6 +27,16 @@ import ( log "github.com/sirupsen/logrus" ) +// HandleRequestAndProxy - Handles the requests and proxies to backend server. +func (rc RequestCall) HandleRequestAndProxy(domainConfig *config.Configuration) { + cached := rc.serveCachedContent() + if !cached { + rc.serveReverseProxy(domainConfig) + } + + logger.LogRequest(*rc.Request, *rc.Response, cached) +} + func getOverridePort(host string, port string, scheme string) string { // if there's already a port it must have priority if strings.Contains(host, ":") { @@ -52,55 +59,31 @@ func getOverridePort(host string, port string, scheme string) string { return portOverride } -// For server requests the URL is parsed from the URI supplied on the -// Request-Line as stored in RequestURI. For most requests, fields other than -// Path and RawQuery will be empty. (See RFC 7230, Section 5.3) -// Ref: https://github.com/golang/go/issues/28940 -func getSchemeFromRequest(req http.Request) string { - if req.TLS != nil { - // TODO: COVERAGE - return "https" - } - return "http" -} +func (rc RequestCall) serveCachedContent() bool { + rcDTO := ConvertToRequestCallDTO(rc) -// FixRequest - Fixes the Request in order to use the load balanced host. -func FixRequest(url url.URL, forwarding config.Forward, req *http.Request) { - scheme := utils.IfEmpty(forwarding.Scheme, getSchemeFromRequest(*req)) - host := utils.IfEmpty(forwarding.Host, url.Host) + uriobj, err := storage.RetrieveCachedContent(rcDTO) + if err != nil { + rc.Response.Header().Set(response.CacheStatusHeader, response.CacheStatusHeaderMiss) - balancedHost := balancer.GetLBRoundRobin(forwarding.Host, url.Host) - overridePort := getOverridePort(balancedHost, forwarding.Port, scheme) + log.Warnf("Error on serving cached content: %s", err) + return false + } - // The value of r.URL.Host and r.Host are almost always different. On a - // proxy server, r.URL.Host is the host of the target server and r.Host is - // the host of the proxy server itself. - // Ref: https://stackoverflow.com/a/42926149/888162 - req.Header.Set("X-Forwarded-Host", req.Header.Get("Host")) + ctx := rc.Request.Context() + transport.ServeCachedResponse(ctx, rc.Response, uriobj) + rc.Response.Header().Set(response.CacheStatusHeader, response.CacheStatusHeaderHit) - req.URL.Host = balancedHost + overridePort - req.URL.Scheme = scheme - req.Host = host + return true } -func serveReverseProxy( - forwarding config.Forward, - target url.URL, - lwr *response.LoggedResponseWriter, - req *http.Request, -) { - domainConfig := config.DomainConf(req.Host) - - FixRequest(target, forwarding, req) - - proxyURL := &url.URL{ - Scheme: req.URL.Scheme, - Host: req.URL.Host, - } +func (rc RequestCall) serveReverseProxy(domainConfig *config.Configuration) { + forwarding := domainConfig.Server.Forwarding + proxyURL := rc.patchRequestForReverseProxy(forwarding) log.Debugf("ProxyURL: %s", proxyURL.String()) - log.Debugf("Req URL: %s", req.URL.String()) - log.Debugf("Req Host: %s", req.Host) + log.Debugf("Req URL: %s", rc.Request.URL.String()) + log.Debugf("Req Host: %s", rc.Request.Host) proxy := httputil.NewSingleHostReverseProxy(proxyURL) // G402 (CWE-295): TLS InsecureSkipVerify may be true. (Confidence: LOW, Severity: HIGH) @@ -110,94 +93,47 @@ func serveReverseProxy( InsecureSkipVerify: domainConfig.Server.Forwarding.InsecureBridge, }, } // #nosec - proxy.ServeHTTP(lwr, req) + proxy.ServeHTTP(rc.Response, rc.Request) - stored, err := storage.StoreGeneratedPage(*req, *lwr) - if !stored || err != nil { - logger.Log(*req, fmt.Sprintf("Not Stored: %v", err)) - } -} + rcDTO := ConvertToRequestCallDTO(rc) -func serveCachedContent( - lwr *response.LoggedResponseWriter, - req http.Request, - url url.URL, -) bool { - uriobj, err := storage.RetrieveCachedContent(lwr, req) - if err != nil { - lwr.Header().Set(response.CacheStatusHeader, response.CacheStatusHeaderMiss) - - log.Warnf("Error on serving cached content: %s", err) - return false - } - - ctx := req.Context() - transport.ServeCachedResponse(ctx, lwr, uriobj, uriobj.URL) - lwr.Header().Set(response.CacheStatusHeader, response.CacheStatusHeaderHit) - - return true -} - -func getListeningPort(ctx context.Context) string { - localAddrContextKey := ctx.Value(http.LocalAddrContextKey) - listeningPort := "" - if localAddrContextKey != nil { - srvAddr := localAddrContextKey.(*net.TCPAddr) - listeningPort = strconv.Itoa(srvAddr.Port) + stored, err := storage.StoreGeneratedPage(rcDTO, domainConfig.Cache) + if !stored || err != nil { + logger.Log(*rc.Request, fmt.Sprintf("Not Stored: %v", err)) } - - return listeningPort } -// HandleRequest - Handles the entrypoint and directs the traffic to the right handler. -func HandleRequest(res http.ResponseWriter, req *http.Request) { - lwr := response.NewLoggedResponseWriter(res) - - listeningPort := getListeningPort(req.Context()) - - domainConfig := config.DomainConf(req.Host) - if domainConfig == nil || - (domainConfig.Server.Port.HTTP != listeningPort && - domainConfig.Server.Port.HTTPS != listeningPort) { - lwr.WriteHeader(http.StatusNotImplemented) - logger.LogRequest(*req, *lwr, false) - log.Errorf("Missing configuration in HandleRequest for %s (listening on :%s).", req.Host, listeningPort) - return - } +// FixRequest - Fixes the Request in order to use the load balanced host. +func (rc *RequestCall) FixRequest(url url.URL, forwarding config.Forward) { + scheme := utils.IfEmpty(forwarding.Scheme, rc.GetScheme()) + host := utils.IfEmpty(forwarding.Host, url.Host) - if getSchemeFromRequest(*req) == "http" && domainConfig.Server.Forwarding.HTTP2HTTPS { - RedirectToHTTPS(lwr.ResponseWriter, req, domainConfig.Server.Forwarding.RedirectStatusCode) - return - } + balancedHost := balancer.GetLBRoundRobin(forwarding.Host, url.Host) + overridePort := getOverridePort(balancedHost, forwarding.Port, scheme) - if req.Method == "PURGE" { - HandlePurge(lwr, req) - return - } + // The value of r.URL.Host and r.Host are almost always different. On a + // proxy server, r.URL.Host is the host of the target server and r.Host is + // the host of the proxy server itself. + // Ref: https://stackoverflow.com/a/42926149/888162 + rc.Request.Header.Set("X-Forwarded-Host", rc.Request.Header.Get("Host")) - if req.Method == http.MethodConnect { - lwr.WriteHeader(http.StatusMethodNotAllowed) - } else { - HandleRequestAndProxy(lwr, req) - } + rc.Request.URL.Host = balancedHost + overridePort + rc.Request.URL.Scheme = scheme + rc.Request.Host = host } -// HandleRequestAndProxy - Handles the requests and proxies to backend server. -func HandleRequestAndProxy(lwr *response.LoggedResponseWriter, req *http.Request) { - domainConfig := config.DomainConf(req.Host) - forwarding := domainConfig.Server.Forwarding +func (rc *RequestCall) patchRequestForReverseProxy(forwarding config.Forward) *url.URL { + overridePort := getOverridePort(forwarding.Host, forwarding.Port, rc.GetScheme()) + targetURL := *rc.Request.URL + targetURL.Scheme = rc.GetScheme() + targetURL.Host = forwarding.Host + overridePort - scheme := utils.IfEmpty(forwarding.Scheme, getSchemeFromRequest(*req)) - overridePort := getOverridePort(forwarding.Host, forwarding.Port, scheme) + rc.FixRequest(targetURL, forwarding) - proxyURL := *req.URL - proxyURL.Scheme = scheme - proxyURL.Host = forwarding.Host + overridePort - - cached := serveCachedContent(lwr, *req, proxyURL) - if !cached { - serveReverseProxy(forwarding, proxyURL, lwr, req) + proxyURL := &url.URL{ + Scheme: rc.Request.URL.Scheme, + Host: rc.Request.URL.Host, } - logger.LogRequest(*req, *lwr, cached) + return proxyURL } diff --git a/server/handler/http_functional_test.go b/server/handler/http_functional_test.go index fbd11632..ea0ce0a7 100644 --- a/server/handler/http_functional_test.go +++ b/server/handler/http_functional_test.go @@ -25,6 +25,7 @@ import ( "github.com/fabiocicerchia/go-proxy-cache/server/balancer" "github.com/fabiocicerchia/go-proxy-cache/server/handler" "github.com/fabiocicerchia/go-proxy-cache/utils" + circuit_breaker "github.com/fabiocicerchia/go-proxy-cache/utils/circuit-breaker" ) func setCommonConfig() { @@ -44,7 +45,7 @@ func setCommonConfig() { Port: "6379", DB: 0, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: time.Duration(1), // clears counts immediately @@ -59,7 +60,7 @@ func TestHTTPEndToEndCallRedirect(t *testing.T) { setCommonConfig() config.Config.Server.Forwarding.Scheme = "http" balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) req, err := http.NewRequest("GET", "/", nil) @@ -92,7 +93,7 @@ func TestHTTPEndToEndCallWithoutCache(t *testing.T) { config.Config.Domains["www.w3.org"] = conf balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) engine.GetConn(config.Config.Server.Forwarding.Host).Close() @@ -130,7 +131,7 @@ func TestHTTPEndToEndCallWithCacheMiss(t *testing.T) { } balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) _, err := engine.GetConn(config.Config.Server.Forwarding.Host).PurgeAll() @@ -177,7 +178,7 @@ func TestHTTPEndToEndCallWithCacheHit(t *testing.T) { AllowedStatuses: []int{200, 301, 302}, AllowedMethods: []string{"HEAD", "GET"}, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: time.Duration(1), // clears counts immediately @@ -186,7 +187,7 @@ func TestHTTPEndToEndCallWithCacheHit(t *testing.T) { } balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) _, _ = engine.GetConn(config.Config.Server.Forwarding.Host).PurgeAll() @@ -285,7 +286,7 @@ func TestHTTPEndToEndCallWithMissingDomain(t *testing.T) { config.Config.Domains["www.w3.org"] = conf balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) engine.GetConn(config.Config.Server.Forwarding.Host).Close() @@ -316,7 +317,7 @@ func TestHTTPSEndToEndCallRedirect(t *testing.T) { config.Config.Server.Forwarding.InsecureBridge = true balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) req, err := http.NewRequest("GET", "/", nil) @@ -349,7 +350,7 @@ func TestHTTPSEndToEndCallWithoutCache(t *testing.T) { config.Config.Domains["www.w3.org"] = conf balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) engine.GetConn(config.Config.Server.Forwarding.Host).Close() @@ -387,7 +388,7 @@ func TestHTTPSEndToEndCallWithCacheMiss(t *testing.T) { } balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) _, err := engine.GetConn(config.Config.Server.Forwarding.Host).PurgeAll() @@ -434,7 +435,7 @@ func TestHTTPSEndToEndCallWithCacheHit(t *testing.T) { AllowedStatuses: []int{200, 301, 302}, AllowedMethods: []string{"HEAD", "GET"}, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: time.Duration(1), // clears counts immediately @@ -443,7 +444,7 @@ func TestHTTPSEndToEndCallWithCacheHit(t *testing.T) { } balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) _, _ = engine.GetConn(config.Config.Server.Forwarding.Host).PurgeAll() @@ -509,7 +510,7 @@ func TestHTTPSEndToEndCallWithMissingDomain(t *testing.T) { config.Config.Domains["www.w3.org"] = conf balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) engine.GetConn(config.Config.Server.Forwarding.Host).Close() diff --git a/server/handler/http_unit_test.go b/server/handler/http_unit_test.go index 07d6dd0a..5cbb817c 100644 --- a/server/handler/http_unit_test.go +++ b/server/handler/http_unit_test.go @@ -50,12 +50,14 @@ func TestFixRequestOneItemInLB(t *testing.T) { } balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - handler.FixRequest(u, config.Config.Server.Forwarding, reqMock) - assert.Equal(t, "localhost", reqMock.Header.Get("X-Forwarded-Host")) + r := handler.RequestCall{Request: reqMock} + r.FixRequest(u, config.Config.Server.Forwarding) - assert.Equal(t, "server1:443", reqMock.URL.Host) - assert.Equal(t, "developer.mozilla.org", reqMock.Host) + assert.Equal(t, "localhost", r.Request.Header.Get("X-Forwarded-Host")) + + assert.Equal(t, "server1:443", r.Request.URL.Host) + assert.Equal(t, "developer.mozilla.org", r.Request.Host) } func TestFixRequestThreeItemsInLB(t *testing.T) { @@ -88,19 +90,20 @@ func TestFixRequestThreeItemsInLB(t *testing.T) { // --- FIRST ROUND - handler.FixRequest(u, config.Config.Server.Forwarding, reqMock) + r := handler.RequestCall{Request: reqMock} + r.FixRequest(u, config.Config.Server.Forwarding) - assert.Equal(t, "localhost", reqMock.Header.Get("X-Forwarded-Host")) + assert.Equal(t, "localhost", r.Request.Header.Get("X-Forwarded-Host")) - assert.Equal(t, "server1:443", reqMock.URL.Host) - assert.Equal(t, "developer.mozilla.org", reqMock.Host) + assert.Equal(t, "server1:443", r.Request.URL.Host) + assert.Equal(t, "developer.mozilla.org", r.Request.Host) // --- SECOND ROUND - handler.FixRequest(u, config.Config.Server.Forwarding, reqMock) + r.FixRequest(u, config.Config.Server.Forwarding) - assert.Equal(t, "localhost", reqMock.Header.Get("X-Forwarded-Host")) + assert.Equal(t, "localhost", r.Request.Header.Get("X-Forwarded-Host")) - assert.Equal(t, "server2:443", reqMock.URL.Host) - assert.Equal(t, "developer.mozilla.org", reqMock.Host) + assert.Equal(t, "server2:443", r.Request.URL.Host) + assert.Equal(t, "developer.mozilla.org", r.Request.Host) } diff --git a/server/handler/purge.go b/server/handler/purge.go index 46ac0edd..2b0ce352 100644 --- a/server/handler/purge.go +++ b/server/handler/purge.go @@ -12,34 +12,27 @@ package handler import ( "net/http" - "github.com/fabiocicerchia/go-proxy-cache/cache" "github.com/fabiocicerchia/go-proxy-cache/config" - "github.com/fabiocicerchia/go-proxy-cache/server/response" + "github.com/fabiocicerchia/go-proxy-cache/server/storage" "github.com/fabiocicerchia/go-proxy-cache/utils" log "github.com/sirupsen/logrus" ) // HandlePurge - Purges the cache for the requested URI. -func HandlePurge(lwr *response.LoggedResponseWriter, req *http.Request) { - domainConfig := config.DomainConf(req.Host) +func (rc RequestCall) HandlePurge(domainConfig *config.Configuration) { forwarding := domainConfig.Server.Forwarding + scheme := utils.IfEmpty(forwarding.Scheme, rc.GetScheme()) - scheme := utils.IfEmpty(forwarding.Scheme, getSchemeFromRequest(*req)) - - proxyURL := *req.URL - proxyURL.Scheme = scheme - proxyURL.Host = forwarding.Host - - status, err := cache.PurgeFullPage(req.Method, proxyURL) - + rcDTO := ConvertToRequestCallDTO(rc) + status, err := storage.PurgeCachedContent(scheme, forwarding.Host, rcDTO) if !status || err != nil { - lwr.WriteHeader(http.StatusNotFound) - _ = response.WriteBody(lwr, "KO") + rc.Response.WriteHeader(http.StatusNotFound) + _ = rc.Response.WriteBody("KO") - log.Warnf("URL Not Purged %s: %v\n", proxyURL.String(), err) + log.Warnf("URL Not Purged %s: %v\n", rc.Request.URL.String(), err) return } - lwr.WriteHeader(http.StatusOK) - _ = response.WriteBody(lwr, "OK") + rc.Response.WriteHeader(http.StatusOK) + _ = rc.Response.WriteBody("OK") } diff --git a/server/handler/purge_test.go b/server/handler/purge_test.go index b3873ebb..074fcd8a 100644 --- a/server/handler/purge_test.go +++ b/server/handler/purge_test.go @@ -24,6 +24,7 @@ import ( "github.com/fabiocicerchia/go-proxy-cache/server/balancer" "github.com/fabiocicerchia/go-proxy-cache/server/handler" "github.com/fabiocicerchia/go-proxy-cache/utils" + circuit_breaker "github.com/fabiocicerchia/go-proxy-cache/utils/circuit-breaker" "github.com/stretchr/testify/assert" ) @@ -46,7 +47,7 @@ func TestEndToEndCallPurgeDoNothing(t *testing.T) { AllowedStatuses: []int{200, 301, 302}, AllowedMethods: []string{"HEAD", "GET"}, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: time.Duration(1), // clears counts immediately @@ -54,7 +55,7 @@ func TestEndToEndCallPurgeDoNothing(t *testing.T) { }, } - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) @@ -102,7 +103,7 @@ func TestEndToEndCallPurge(t *testing.T) { AllowedStatuses: []int{200, 301, 302}, AllowedMethods: []string{"HEAD", "GET"}, }, - CircuitBreaker: config.CircuitBreaker{ + CircuitBreaker: circuit_breaker.CircuitBreaker{ Threshold: 2, // after 2nd request, if meet FailureRate goes open. FailureRate: 0.5, // 1 out of 2 fails, or more Interval: time.Duration(1), // clears counts immediately @@ -112,7 +113,7 @@ func TestEndToEndCallPurge(t *testing.T) { balancer.InitRoundRobin(config.Config.Server.Forwarding.Host, config.Config.Server.Forwarding.Endpoints) - config.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) + circuit_breaker.InitCircuitBreaker(config.Config.Server.Forwarding.Host, config.Config.CircuitBreaker) engine.InitConn(config.Config.Server.Forwarding.Host, config.Config.Cache) diff --git a/server/handler/redirect.go b/server/handler/redirect.go index 142b52e6..46a2fa1f 100644 --- a/server/handler/redirect.go +++ b/server/handler/redirect.go @@ -16,14 +16,14 @@ import ( ) // RedirectToHTTPS - Redirects from HTTP to HTTPS. -func RedirectToHTTPS(w http.ResponseWriter, req *http.Request, redirectStatusCode int) { - targetURL := req.URL +func (rc RequestCall) RedirectToHTTPS(redirectStatusCode int) { + targetURL := rc.Request.URL targetURL.Scheme = "https" - targetURL.Host = req.Host + targetURL.Host = rc.Request.Host target := targetURL.String() log.Infof("Redirect to: %s", target) - http.Redirect(w, req, target, redirectStatusCode) + http.Redirect(rc.Response, rc.Request, target, redirectStatusCode) } diff --git a/server/logger/log.go b/server/logger/log.go index 1f005ec6..a3e66022 100644 --- a/server/logger/log.go +++ b/server/logger/log.go @@ -21,6 +21,7 @@ import ( "github.com/fabiocicerchia/go-proxy-cache/config" "github.com/fabiocicerchia/go-proxy-cache/server/response" "github.com/fabiocicerchia/go-proxy-cache/utils" + "github.com/fabiocicerchia/go-proxy-cache/utils/slice" ) // Log - Logs against a requested URL. @@ -53,7 +54,7 @@ func LogRequest(req http.Request, lwr response.LoggedResponseWriter, cached bool `$request_method`, method, `$request`, req.URL.String(), `$status`, strconv.Itoa(lwr.StatusCode), - `$body_bytes_sent`, strconv.Itoa(utils.LenSliceBytes(lwr.Content)), + `$body_bytes_sent`, strconv.Itoa(slice.LenSliceBytes(lwr.Content)), `$http_referer`, req.Referer(), `$http_user_agent`, req.UserAgent(), `$cached_status`, fmt.Sprintf("%v", cached), diff --git a/server/response/response.go b/server/response/response.go index c3f24c78..52d02440 100644 --- a/server/response/response.go +++ b/server/response/response.go @@ -58,18 +58,18 @@ func (lwr *LoggedResponseWriter) Write(p []byte) (int, error) { } // CopyHeaders - Adds the headers to the response. -func CopyHeaders(dst http.Header, src http.Header) { +func (lwr *LoggedResponseWriter) CopyHeaders(src http.Header) { for k, vv := range src { for _, v := range vv { - dst.Add(k, v) + lwr.Header().Add(k, v) } } } // WriteBody - Sends the body to the client. -func WriteBody(rw http.ResponseWriter, page string) bool { +func (lwr *LoggedResponseWriter) WriteBody(page string) bool { pageByte := []byte(page) - sent, err := rw.Write(pageByte) + sent, err := lwr.ResponseWriter.Write(pageByte) return sent > 0 && err == nil } diff --git a/server/server.go b/server/server.go index e7c96729..a2e67155 100644 --- a/server/server.go +++ b/server/server.go @@ -25,15 +25,46 @@ import ( "github.com/fabiocicerchia/go-proxy-cache/server/handler" "github.com/fabiocicerchia/go-proxy-cache/server/logger" srvtls "github.com/fabiocicerchia/go-proxy-cache/server/tls" + circuitbreaker "github.com/fabiocicerchia/go-proxy-cache/utils/circuit-breaker" ) +// Servers - Contains the HTTP/HTTPS servers. type Servers struct { HTTP map[string]*http.Server HTTPS map[string]*http.Server } -// CreateServerConfig - Generates the http.Server configuration. -func CreateServerConfig(domain string, port string) *http.Server { +// Run - Starts the GoProxyCache servers' listeners. +func Run(configFile string) { + // Init configs + config.InitConfigFromFileOrEnv(configFile) + config.Print() + + servers := &Servers{ + HTTP: make(map[string]*http.Server), + HTTPS: make(map[string]*http.Server), + } + for _, domain := range config.GetDomains() { + servers.StartDomainServer(domain) + } + + // start server http & https + servers.startListeners() + + // Wait for an interrupt + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + <-c + + // Attempt a graceful shutdown + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + servers.shutdownServers(ctx) +} + +// InitServer - Generates the http.Server configuration. +func InitServer(domain string) *http.Server { // THIS IS FOR EVERY DOMAIN, NO DOMAIN OVERRIDE. timeout := config.Config.Server.Timeout gzip := config.Config.Server.GZip @@ -58,7 +89,6 @@ func CreateServerConfig(domain string, port string) *http.Server { } server := &http.Server{ - Addr: ":" + port, ReadTimeout: time.Duration(timeout.Read) * time.Second, WriteTimeout: time.Duration(timeout.Write) * time.Second, IdleTimeout: time.Duration(timeout.Idle) * time.Second, @@ -69,21 +99,28 @@ func CreateServerConfig(domain string, port string) *http.Server { return server } -// GetServerConfigs - Returns a http.Server configuration for HTTP and HTTPS. -func (s *Servers) AddServerConfigs(domain string, domainConfig *config.Configuration) { - srvHTTP := CreateServerConfig(domain, domainConfig.Server.Port.HTTP) +// AttachPlain - Adds a new HTTP server in the listener container. +func (s *Servers) AttachPlain(port string, server *http.Server) { + s.HTTP[port] = server + s.HTTP[port].Addr = ":" + port +} - srvHTTPS := CreateServerConfig(domain, domainConfig.Server.Port.HTTPS) - srvtls.ServerOverrides(domain, srvHTTPS, &srvtls.CertificatePair{ - Cert: domainConfig.Server.TLS.CertFile, - Key: domainConfig.Server.TLS.KeyFile, - }) +// AttachSecure - Adds a new HTTPS server in the listener container. +func (s *Servers) AttachSecure(port string, server *http.Server) { + s.HTTPS[port] = server + s.HTTPS[port].Addr = ":" + port +} + +// InitServers - Returns a http.Server configuration for HTTP and HTTPS. +func (s *Servers) InitServers(domain string, domainInitServer config.Server) { + srv := InitServer(domain) + s.AttachPlain(domainInitServer.Port.HTTP, srv) - s.HTTP[domainConfig.Server.Port.HTTP] = srvHTTP - s.HTTPS[domainConfig.Server.Port.HTTPS] = srvHTTPS + srvHTTPS := srvtls.ServerOverrides(domain, *srv, domainInitServer) + s.AttachSecure(domainInitServer.Port.HTTPS, &srvHTTPS) } -// StartDomainServer - Configures and start listinening for a particular domain. +// StartDomainServer - Configures and start listening for a particular domain. func (s *Servers) StartDomainServer(domain string) { domainConfig := config.DomainConf(domain) if domainConfig == nil { @@ -92,57 +129,36 @@ func (s *Servers) StartDomainServer(domain string) { } // redis connect - config.InitCircuitBreaker(domain, domainConfig.CircuitBreaker) + circuitbreaker.InitCircuitBreaker(domain, domainConfig.CircuitBreaker) engine.InitConn(domain, domainConfig.Cache) // Log setup values logger.LogSetup(domainConfig.Server) // config server http & https - s.AddServerConfigs(domain, domainConfig) + s.InitServers(domain, domainConfig.Server) // lb balancer.InitRoundRobin(domain, domainConfig.Server.Forwarding.Endpoints) } -// Start the GoProxyCache server. -func Start(configFile string) { - // Init configs - config.InitConfigFromFileOrEnv(configFile) - config.Print() - - servers := &Servers{ - HTTP: make(map[string]*http.Server), - HTTPS: make(map[string]*http.Server), - } - for _, domain := range config.GetDomains() { - servers.StartDomainServer(domain) - } - - // start server http & https - for _, srvHTTP := range servers.HTTP { +func (s Servers) startListeners() { + for _, srvHTTP := range s.HTTP { go func(srv *http.Server) { log.Fatal(srv.ListenAndServe()) }(srvHTTP) } - for _, srvHTTPS := range servers.HTTPS { + for _, srvHTTPS := range s.HTTPS { go func(srv *http.Server) { log.Fatal(srv.ListenAndServeTLS("", "")) }(srvHTTPS) } +} - // Wait for an interrupt - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - <-c - - // Attempt a graceful shutdown - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - for k, v := range servers.HTTP { +func (s Servers) shutdownServers(ctx context.Context) { + for k, v := range s.HTTP { err := v.Shutdown(ctx) if err != nil { log.Fatalf("Cannot shutdown server %s: %s", k, err) } } - for k, v := range servers.HTTPS { + for k, v := range s.HTTPS { err := v.Shutdown(ctx) if err != nil { log.Fatalf("Cannot shutdown server %s: %s", k, err) diff --git a/server/storage/storage.go b/server/storage/storage.go index 508a7963..7e12940f 100644 --- a/server/storage/storage.go +++ b/server/storage/storage.go @@ -10,7 +10,6 @@ package storage // Repo: https://github.com/fabiocicerchia/go-proxy-cache import ( - "fmt" "net/http" log "github.com/sirupsen/logrus" @@ -18,55 +17,61 @@ import ( "github.com/fabiocicerchia/go-proxy-cache/cache" "github.com/fabiocicerchia/go-proxy-cache/config" "github.com/fabiocicerchia/go-proxy-cache/server/response" - "github.com/fabiocicerchia/go-proxy-cache/utils" + "github.com/fabiocicerchia/go-proxy-cache/utils/ttl" ) +// RequestCallDTO - DTO object containing request and response. +type RequestCallDTO struct { + Response response.LoggedResponseWriter + Request http.Request + CacheObj cache.CacheObj +} + // RetrieveCachedContent - Retrives the cached response. -func RetrieveCachedContent( - lwr *response.LoggedResponseWriter, - req http.Request, -) (cache.URIObj, error) { - method := req.Method - reqHeaders := req.Header +func RetrieveCachedContent(rc RequestCallDTO) (cache.URIObj, error) { + method := rc.Request.Method + reqHeaders := rc.Request.Header - url := *req.URL - url.Host = req.Host + url := *rc.Request.URL + url.Host = rc.Request.Host - uriobj, err := cache.RetrieveFullPage(method, url, reqHeaders) + err := rc.CacheObj.RetrieveFullPage(method, url, reqHeaders) if err != nil { log.Warnf("Cannot retrieve page %s: %s\n", url.String(), err) } - if !cache.IsStatusAllowed(uriobj.StatusCode) || utils.LenSliceBytes(uriobj.Content) == 0 { - return uriobj, fmt.Errorf( - "not allowed. status %d - content length %d", - uriobj.StatusCode, - utils.LenSliceBytes(uriobj.Content), - ) + ok, err := rc.CacheObj.IsValid() + if !ok || err != nil { + return cache.URIObj{}, err } - return uriobj, nil + return rc.CacheObj.CurrentObj, nil } // StoreGeneratedPage - Stores a response in the cache. -func StoreGeneratedPage( - req http.Request, - lwr response.LoggedResponseWriter, -) (bool, error) { - domainConfig := config.DomainConf(req.Host) - ttl := utils.GetTTL(lwr.Header(), domainConfig.Cache.TTL) - - response := cache.URIObj{ - URL: *req.URL, - Host: req.Host, - Method: req.Method, - StatusCode: lwr.StatusCode, - RequestHeaders: req.Header, - ResponseHeaders: lwr.Header(), - Content: lwr.Content, +func StoreGeneratedPage(rc RequestCallDTO, domainConfigCache config.Cache) (bool, error) { + ttl := ttl.GetTTL(rc.Response.Header(), domainConfigCache.TTL) + + rc.CacheObj.CurrentObj = cache.URIObj{ + URL: *rc.Request.URL, + Host: rc.Request.Host, + Method: rc.Request.Method, + StatusCode: rc.Response.StatusCode, + RequestHeaders: rc.Request.Header, + ResponseHeaders: rc.Response.Header(), + Content: rc.Response.Content, } - done, err := cache.StoreFullPage(response, ttl) + done, err := rc.CacheObj.StoreFullPage(ttl) return done, err } + +// PurgeCachedContent - Purges a content in the cache. +func PurgeCachedContent(scheme string, host string, rc RequestCallDTO) (bool, error) { + proxyURL := *rc.Request.URL + proxyURL.Scheme = scheme + proxyURL.Host = host + + return rc.CacheObj.PurgeFullPage(rc.Request.Method, proxyURL) +} diff --git a/server/tls/tls.go b/server/tls/tls.go index 42361b43..4b029e23 100644 --- a/server/tls/tls.go +++ b/server/tls/tls.go @@ -21,12 +21,6 @@ import ( "golang.org/x/crypto/acme/autocert" ) -// CertificatePair - Pair of certificate and key. -type CertificatePair struct { - Cert string - Key string -} - var httpsDomains []string var certificates map[string]*crypto_tls.Certificate var tlsConfig *crypto_tls.Config @@ -34,25 +28,27 @@ var tlsConfig *crypto_tls.Config // ServerOverrides - Overrides the http.Server configuration for TLS. func ServerOverrides( domain string, - server *http.Server, - certPair *CertificatePair, -) { + server http.Server, + domainInitServer config.Server, +) http.Server { + newServer := server var err error - domainConfig := config.DomainConf(domain) - tlsConfig, err = Config(domain, certPair.Cert, certPair.Key) + tlsConfig, err = Config(domain, domainInitServer.TLS.CertFile, domainInitServer.TLS.KeyFile) if err != nil { log.Fatal(err) - return + return newServer } - server.TLSConfig = tlsConfig + newServer.TLSConfig = tlsConfig // TODO: check this: server.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0), - if domainConfig.Server.TLS.Auto { - certManager := InitCertManager(domainConfig.Server.Forwarding.Host, domainConfig.Server.TLS.Email) + if domainInitServer.TLS.Auto { + certManager := InitCertManager(domainInitServer.Forwarding.Host, domainInitServer.TLS.Email) - server.TLSConfig = certManager.TLSConfig() + newServer.TLSConfig = certManager.TLSConfig() } + + return newServer } // Config - Returns a TLS configuration. diff --git a/server/transport/http.go b/server/transport/http.go index 4565c1c0..75fdd57d 100644 --- a/server/transport/http.go +++ b/server/transport/http.go @@ -13,7 +13,6 @@ import ( "context" "io" "net/http" - "net/url" "strings" "github.com/fabiocicerchia/go-proxy-cache/cache" @@ -73,15 +72,8 @@ func shouldPanicOnCopyError(ctx context.Context) bool { } // ServeCachedResponse - Serve a cached response. -func ServeCachedResponse( - ctx context.Context, - lwr *response.LoggedResponseWriter, - uriobj cache.URIObj, - url url.URL, -) { - var ctxWC context.Context - var cancel context.CancelFunc - ctxWC, cancel = context.WithCancel(ctx) +func ServeCachedResponse(ctx context.Context, lwr *response.LoggedResponseWriter, uriobj cache.URIObj) { + ctxWC, cancel := context.WithCancel(ctx) defer cancel() go func() { select { @@ -96,12 +88,17 @@ func ServeCachedResponse( Header: uriobj.ResponseHeaders, } - // HTTP Headers + announcedTrailers := handleHeaders(lwr, res) + handleBody(ctx, lwr, uriobj.Content) + handleTrailer(announcedTrailers, lwr, res) +} + +func handleHeaders(lwr *response.LoggedResponseWriter, res http.Response) int { removeConnectionHeaders(res.Header) for _, h := range HopHeaders { res.Header.Del(h) } - response.CopyHeaders(lwr.Header(), res.Header) + lwr.CopyHeaders(res.Header) // The "Trailer" header isn't included in the Transport's response, // at least for *http.Transport. Build it up from Trailer. @@ -116,7 +113,11 @@ func ServeCachedResponse( lwr.WriteHeader(res.StatusCode) - err := copyResponse(lwr, uriobj.Content) + return announcedTrailers +} + +func handleBody(ctx context.Context, lwr *response.LoggedResponseWriter, content [][]byte) { + err := copyResponse(lwr, content) if err != nil { // Since we're streaming the response, if we run into an error all we can do // is abort the request. Issue 23643: ReverseProxy should use ErrAbortHandler @@ -127,8 +128,6 @@ func ServeCachedResponse( } panic(http.ErrAbortHandler) } - - handleTrailer(announcedTrailers, lwr, res) } func handleTrailer(announcedTrailers int, lwr *response.LoggedResponseWriter, res http.Response) { @@ -142,7 +141,7 @@ func handleTrailer(announcedTrailers int, lwr *response.LoggedResponseWriter, re } if len(res.Trailer) == announcedTrailers { - response.CopyHeaders(lwr.Header(), res.Trailer) + lwr.CopyHeaders(res.Trailer) return } diff --git a/utils/base64.go b/utils/base64/base64.go similarity index 77% rename from utils/base64.go rename to utils/base64/base64.go index 48acb2f2..4fd0283d 100644 --- a/utils/base64.go +++ b/utils/base64/base64.go @@ -1,4 +1,4 @@ -package utils +package base64 // __ // .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. @@ -13,12 +13,12 @@ import ( "encoding/base64" ) -// Base64Encode - Encodes object with base64. -func Base64Encode(source []byte) string { +// Encode - Encodes object with base64. +func Encode(source []byte) string { return base64.StdEncoding.EncodeToString(source) } -// Base64Decode - Decodes object with base64. -func Base64Decode(source string) ([]byte, error) { +// Decode - Decodes object with base64. +func Decode(source string) ([]byte, error) { return base64.StdEncoding.DecodeString(source) } diff --git a/utils/base64_test.go b/utils/base64/base64_test.go similarity index 73% rename from utils/base64_test.go rename to utils/base64/base64_test.go index 8ae771a1..696a27d7 100644 --- a/utils/base64_test.go +++ b/utils/base64/base64_test.go @@ -1,6 +1,6 @@ -// +build unit +// +build all unit -package utils_test +package base64_test // __ // .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. @@ -14,15 +14,15 @@ package utils_test import ( "testing" - "github.com/fabiocicerchia/go-proxy-cache/utils" + "github.com/fabiocicerchia/go-proxy-cache/utils/base64" "github.com/stretchr/testify/assert" ) -func TestBase64EncodeDecode(t *testing.T) { +func TestEncodeDecode(t *testing.T) { str := []byte("test string") - encoded := utils.Base64Encode(str) - decoded, err := utils.Base64Decode(encoded) + encoded := base64.Encode(str) + decoded, err := base64.Decode(encoded) assert.Nil(t, err) assert.Equal(t, str, decoded) @@ -31,8 +31,8 @@ func TestBase64EncodeDecode(t *testing.T) { func TestBase64CorruptedDecode(t *testing.T) { str := []byte("test string") - encoded := utils.Base64Encode(str) - decoded, err := utils.Base64Decode(encoded) + encoded := base64.Encode(str) + decoded, err := base64.Decode(encoded) assert.Nil(t, err) assert.Equal(t, str, decoded) diff --git a/utils/circuit-breaker/circuit-breaker.go b/utils/circuit-breaker/circuit-breaker.go new file mode 100644 index 00000000..d18ec2e1 --- /dev/null +++ b/utils/circuit-breaker/circuit-breaker.go @@ -0,0 +1,62 @@ +package circuitbreaker + +// __ +// .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. +// | _ | _ |______| _ | _| _ |_ _| | |______| __| _ | __| | -__| +// |___ |_____| | __|__| |_____|__.__|___ | |____|___._|____|__|__|_____| +// |_____| |__| |_____| +// +// Copyright (c) 2020 Fabio Cicerchia. https://fabiocicerchia.it. MIT License +// Repo: https://github.com/fabiocicerchia/go-proxy-cache + +import ( + "time" + + log "github.com/sirupsen/logrus" + + "github.com/sony/gobreaker" +) + +var cb map[string]*gobreaker.CircuitBreaker + +// CircuitBreaker - Settings for redis circuit breaker. +type CircuitBreaker struct { + Threshold uint32 + FailureRate float64 + Interval time.Duration + Timeout time.Duration + MaxRequests uint32 +} + +// InitCircuitBreaker - Initialise the Circuit Breaker. +func InitCircuitBreaker(name string, config CircuitBreaker) { + st := gobreaker.Settings{ + Name: name, + MaxRequests: config.MaxRequests, + Interval: config.Interval, + Timeout: config.Timeout, + ReadyToTrip: func(counts gobreaker.Counts) bool { + failureRatio := float64(counts.TotalFailures) / float64(counts.Requests) + return counts.Requests >= config.Threshold && failureRatio >= config.FailureRate + }, + OnStateChange: func(name string, from gobreaker.State, to gobreaker.State) { + log.Warnf("Circuit Breaker - Changed from %s to %s", from.String(), to.String()) + }, + } + + if cb == nil { + cb = make(map[string]*gobreaker.CircuitBreaker) + } + + cb[name] = gobreaker.NewCircuitBreaker(st) +} + +// CB - Returns instance of gobreaker.CircuitBreaker. +func CB(name string) *gobreaker.CircuitBreaker { + if val, ok := cb[name]; ok { + return val + } + + log.Warnf("Missing circuit breaker for %s", name) + return nil +} diff --git a/utils/convert.go b/utils/convert/convert.go similarity index 68% rename from utils/convert.go rename to utils/convert/convert.go index d1019c2e..4389b6c2 100644 --- a/utils/convert.go +++ b/utils/convert/convert.go @@ -1,4 +1,4 @@ -package utils +package convert // __ // .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. @@ -14,8 +14,8 @@ import ( "time" ) -// ConvertToDuration - Converts a string to time.Duration -func ConvertToDuration(value string) time.Duration { +// ToDuration - Converts a string to time.Duration +func ToDuration(value string) time.Duration { duration, err := time.ParseDuration(value) if err != nil { return time.Duration(0) @@ -23,17 +23,17 @@ func ConvertToDuration(value string) time.Duration { return duration } -// ConvertToInt - Converts a string to int -func ConvertToInt(value string) int { +// ToInt - Converts a string to int +func ToInt(value string) int { val, _ := strconv.Atoi(value) return val } -// ConvertToIntSlice - Converts a slice of strings to a slice of ints -func ConvertToIntSlice(value []string) []int { +// ToIntSlice - Converts a slice of strings to a slice of ints +func ToIntSlice(value []string) []int { values := []int{} for _, v := range value { - values = append(values, ConvertToInt(v)) + values = append(values, ToInt(v)) } return values } diff --git a/utils/convert/convert_test.go b/utils/convert/convert_test.go new file mode 100644 index 00000000..715dc48f --- /dev/null +++ b/utils/convert/convert_test.go @@ -0,0 +1,84 @@ +// +build all unit + +package convert_test + +// __ +// .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. +// | _ | _ |______| _ | _| _ |_ _| | |______| __| _ | __| | -__| +// |___ |_____| | __|__| |_____|__.__|___ | |____|___._|____|__|__|_____| +// |_____| |__| |_____| +// +// Copyright (c) 2020 Fabio Cicerchia. https://fabiocicerchia.it. MIT License +// Repo: https://github.com/fabiocicerchia/go-proxy-cache + +import ( + "testing" + "time" + + "github.com/fabiocicerchia/go-proxy-cache/utils/convert" + "github.com/stretchr/testify/assert" +) + +// --- ToDuration + +func TestToDurationEmpty(t *testing.T) { + value := convert.ToDuration("") + + assert.Equal(t, time.Duration(0), value) +} + +func TestToDurationSeconds(t *testing.T) { + value := convert.ToDuration("10s") + + assert.Equal(t, time.Duration(10*time.Second), value) +} + +func TestToDurationDifferentValues(t *testing.T) { + value := convert.ToDuration("10m") + assert.Equal(t, time.Duration(10*time.Minute), value) + + value = convert.ToDuration("10h") + assert.Equal(t, time.Duration(10*time.Hour), value) +} + +// --- ToInt + +func TestToIntEmpty(t *testing.T) { + value := convert.ToInt("") + + assert.Equal(t, 0, value) +} + +func TestToIntInvalid(t *testing.T) { + value := convert.ToInt("A") + + assert.Equal(t, 0, value) +} + +func TestToIntValid(t *testing.T) { + value := convert.ToInt("123") + + assert.Equal(t, 123, value) +} + +// --- ToIntSlice + +func TestToIntSliceEmpty(t *testing.T) { + value := convert.ToIntSlice([]string{}) + assert.Equal(t, []int{}, value) + + value = convert.ToIntSlice([]string{""}) + assert.Equal(t, []int{0}, value) +} + +func TestToIntSliceInvalid(t *testing.T) { + value := convert.ToIntSlice([]string{"A"}) + + assert.Equal(t, []int{0}, value) +} + +func TestToIntSliceValid(t *testing.T) { + value := convert.ToIntSlice([]string{"123", "345"}) + + assert.Equal(t, []int{123, 345}, value) +} diff --git a/utils/convert_test.go b/utils/convert_test.go deleted file mode 100644 index 6d133246..00000000 --- a/utils/convert_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// +build unit - -package utils_test - -// __ -// .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. -// | _ | _ |______| _ | _| _ |_ _| | |______| __| _ | __| | -__| -// |___ |_____| | __|__| |_____|__.__|___ | |____|___._|____|__|__|_____| -// |_____| |__| |_____| -// -// Copyright (c) 2020 Fabio Cicerchia. https://fabiocicerchia.it. MIT License -// Repo: https://github.com/fabiocicerchia/go-proxy-cache - -import ( - "testing" - "time" - - "github.com/fabiocicerchia/go-proxy-cache/utils" - "github.com/stretchr/testify/assert" -) - -// --- ConvertToDuration - -func TestConvertToDurationEmpty(t *testing.T) { - value := utils.ConvertToDuration("") - - assert.Equal(t, time.Duration(0), value) - - tearDown() -} - -func TestConvertToDurationSeconds(t *testing.T) { - value := utils.ConvertToDuration("10s") - - assert.Equal(t, time.Duration(10*time.Second), value) - - tearDown() -} - -func TestConvertToDurationDifferentValues(t *testing.T) { - value := utils.ConvertToDuration("10m") - assert.Equal(t, time.Duration(10*time.Minute), value) - - value = utils.ConvertToDuration("10h") - assert.Equal(t, time.Duration(10*time.Hour), value) - - tearDown() -} - -// --- ConvertToInt - -func TestConvertToIntEmpty(t *testing.T) { - value := utils.ConvertToInt("") - - assert.Equal(t, 0, value) - - tearDown() -} - -func TestConvertToIntInvalid(t *testing.T) { - value := utils.ConvertToInt("A") - - assert.Equal(t, 0, value) - - tearDown() -} - -func TestConvertToIntValid(t *testing.T) { - value := utils.ConvertToInt("123") - - assert.Equal(t, 123, value) - - tearDown() -} - -// --- ConvertToIntSlice - -func TestConvertToIntSliceEmpty(t *testing.T) { - value := utils.ConvertToIntSlice([]string{}) - assert.Equal(t, []int{}, value) - - value = utils.ConvertToIntSlice([]string{""}) - assert.Equal(t, []int{0}, value) - - tearDown() -} - -func TestConvertToIntSliceInvalid(t *testing.T) { - value := utils.ConvertToIntSlice([]string{"A"}) - - assert.Equal(t, []int{0}, value) - - tearDown() -} - -func TestConvertToIntSliceValid(t *testing.T) { - value := utils.ConvertToIntSlice([]string{"123", "345"}) - - assert.Equal(t, []int{123, 345}, value) - - tearDown() -} diff --git a/utils/msgpack.go b/utils/msgpack/msgpack.go similarity index 79% rename from utils/msgpack.go rename to utils/msgpack/msgpack.go index 336930ea..a6437449 100644 --- a/utils/msgpack.go +++ b/utils/msgpack/msgpack.go @@ -1,4 +1,4 @@ -package utils +package msgpack // __ // .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. @@ -17,8 +17,8 @@ import ( var msgpackHandler codec.MsgpackHandle -// MsgpackEncode - Encodes object with msgpack. -func MsgpackEncode(obj interface{}) ([]byte, error) { +// Encode - Encodes object with msgpack. +func Encode(obj interface{}) ([]byte, error) { buff := new(bytes.Buffer) encoder := codec.NewEncoder(buff, &msgpackHandler) err := encoder.Encode(obj) @@ -26,8 +26,8 @@ func MsgpackEncode(obj interface{}) ([]byte, error) { return buff.Bytes(), err } -// MsgpackDecode - Decodes object with msgpack. -func MsgpackDecode(b []byte, v interface{}) error { +// Decode - Decodes object with msgpack. +func Decode(b []byte, v interface{}) error { decoder := codec.NewDecoderBytes(b, &msgpackHandler) return decoder.Decode(v) diff --git a/utils/msgpack_test.go b/utils/msgpack/msgpack_test.go similarity index 76% rename from utils/msgpack_test.go rename to utils/msgpack/msgpack_test.go index 9b62a647..bbe0a652 100644 --- a/utils/msgpack_test.go +++ b/utils/msgpack/msgpack_test.go @@ -1,6 +1,6 @@ -// +build unit +// +build all unit -package utils_test +package msgpack_test // __ // .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. @@ -14,18 +14,18 @@ package utils_test import ( "testing" - "github.com/fabiocicerchia/go-proxy-cache/utils" + "github.com/fabiocicerchia/go-proxy-cache/utils/msgpack" "github.com/stretchr/testify/assert" ) -func TestMsgpackEncodeDecode(t *testing.T) { +func TestEncodeDecode(t *testing.T) { str := []byte("test string") - encoded, err := utils.MsgpackEncode(str) + encoded, err := msgpack.Encode(str) assert.Nil(t, err) var decoded []byte - err = utils.MsgpackDecode(encoded, &decoded) + err = msgpack.Decode(encoded, &decoded) assert.Nil(t, err) assert.Equal(t, str, decoded) diff --git a/utils/slice.go b/utils/slice/slice.go similarity index 99% rename from utils/slice.go rename to utils/slice/slice.go index 90556f0a..5f3e691f 100644 --- a/utils/slice.go +++ b/utils/slice/slice.go @@ -1,4 +1,4 @@ -package utils +package slice import ( "net/http" diff --git a/utils/slice_test.go b/utils/slice/slice_test.go similarity index 71% rename from utils/slice_test.go rename to utils/slice/slice_test.go index 7f14939b..2d46d787 100644 --- a/utils/slice_test.go +++ b/utils/slice/slice_test.go @@ -1,6 +1,6 @@ -// +build unit +// +build all unit -package utils_test +package slice_test // __ // .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. @@ -14,120 +14,96 @@ package utils_test import ( "testing" - "github.com/fabiocicerchia/go-proxy-cache/utils" + "github.com/fabiocicerchia/go-proxy-cache/utils/slice" "github.com/stretchr/testify/assert" ) // --- ContainsInt func TestContainsIntEmpty(t *testing.T) { - match := utils.ContainsInt([]int{}, 1) + match := slice.ContainsInt([]int{}, 1) assert.False(t, match) - - tearDown() } func TestContainsIntNoValue(t *testing.T) { - match := utils.ContainsInt([]int{1, 2, 3}, 4) + match := slice.ContainsInt([]int{1, 2, 3}, 4) assert.False(t, match) - - tearDown() } func TestContainsIntValue(t *testing.T) { - match := utils.ContainsInt([]int{1, 2, 3}, 3) + match := slice.ContainsInt([]int{1, 2, 3}, 3) assert.True(t, match) - - tearDown() } // --- ContainsString func TestContainsStringEmpty(t *testing.T) { - match := utils.ContainsString([]string{}, "d") + match := slice.ContainsString([]string{}, "d") assert.False(t, match) - - tearDown() } func TestContainsStringNoValue(t *testing.T) { - match := utils.ContainsString([]string{"a", "b", "c"}, "d") + match := slice.ContainsString([]string{"a", "b", "c"}, "d") assert.False(t, match) - - tearDown() } func TestContainsStringValue(t *testing.T) { - match := utils.ContainsString([]string{"a", "b", "c"}, "c") + match := slice.ContainsString([]string{"a", "b", "c"}, "c") assert.True(t, match) - - tearDown() } // --- Unique func TestUniqueEmpty(t *testing.T) { input := []string{} - value := utils.Unique(input) + value := slice.Unique(input) assert.Equal(t, []string{}, value) - - tearDown() } func TestUniqueOneElement(t *testing.T) { input := []string{"a"} - value := utils.Unique(input) + value := slice.Unique(input) assert.Equal(t, []string{"a"}, value) - - tearDown() } func TestUniqueTwoElements(t *testing.T) { input := []string{"a", "b"} - value := utils.Unique(input) + value := slice.Unique(input) assert.Equal(t, []string{"a", "b"}, value) - - tearDown() } func TestUniqueTwoElementsWithDuplicates(t *testing.T) { input := []string{"a", "b", "c", "b", "a"} - value := utils.Unique(input) + value := slice.Unique(input) assert.Equal(t, []string{"a", "b", "c"}, value) - - tearDown() } // --- LenSliceBytes func TestLenSliceByteEmpty(t *testing.T) { input := [][]byte{} - value := utils.LenSliceBytes(input) + value := slice.LenSliceBytes(input) assert.Equal(t, 0, value) - - tearDown() } func TestLenSliceBytesOneItem(t *testing.T) { input := make([][]byte, 0) input = append(input, []byte("testing")) - value := utils.LenSliceBytes(input) + value := slice.LenSliceBytes(input) assert.Equal(t, 7, value) - - tearDown() } func TestLenSliceBytesTwosItems(t *testing.T) { @@ -135,9 +111,7 @@ func TestLenSliceBytesTwosItems(t *testing.T) { input = append(input, []byte("testing")) input = append(input, []byte("sample")) - value := utils.LenSliceBytes(input) + value := slice.LenSliceBytes(input) assert.Equal(t, 13, value) - - tearDown() } diff --git a/utils/ttl.go b/utils/ttl/ttl.go similarity index 56% rename from utils/ttl.go rename to utils/ttl/ttl.go index 0fabab50..45347c36 100644 --- a/utils/ttl.go +++ b/utils/ttl/ttl.go @@ -1,4 +1,4 @@ -package utils +package ttl // __ // .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. @@ -15,42 +15,61 @@ import ( "strconv" "strings" "time" + + "github.com/fabiocicerchia/go-proxy-cache/utils/slice" ) +func ttlFromExpires(expiresValue string) *time.Duration { + expiresDate, err := http.ParseTime(expiresValue) + if err == nil { + diff := expiresDate.UTC().Sub(time.Now().UTC()) + if diff > 0 { + return &diff + } + } + + return nil +} + +func ttlFromCacheControlChain(cacheControlValue string) *time.Duration { + // Ref: https://tools.ietf.org/html/rfc7234#section-4.2.1 + if strings.Contains(cacheControlValue, "no-cache") || strings.Contains(cacheControlValue, "no-store") { + zeroDuration := time.Duration(0) + return &zeroDuration + } + + if smaxage := GetTTLFromCacheControl("s-maxage", cacheControlValue); smaxage > 0 { + return &smaxage + } + + if maxage := GetTTLFromCacheControl("max-age", cacheControlValue); maxage > 0 { + return &maxage + } + + return nil +} + // GetTTL - Retrieves TTL is seconds from Expires and Cache-Control HTTP headers. func GetTTL(headers http.Header, defaultTTL int) time.Duration { ttl := time.Duration(defaultTTL) * time.Second - expires := GetByKeyCaseInsensitive(headers, "Expires") + expires := slice.GetByKeyCaseInsensitive(headers, "Expires") if expires != nil { expiresValue := expires.([]string)[0] - - expiresDate, err := http.ParseTime(expiresValue) - if err == nil { - diff := expiresDate.UTC().Sub(time.Now().UTC()) - if diff > 0 { - ttl = time.Duration(diff) - } + expiresTTL := ttlFromExpires(expiresValue) + if expiresTTL != nil { + ttl = *expiresTTL } } - cacheControl := GetByKeyCaseInsensitive(headers, "Cache-Control") + cacheControl := slice.GetByKeyCaseInsensitive(headers, "Cache-Control") if cacheControl != nil { cacheControlValue := strings.ToLower(cacheControl.([]string)[0]) - - // Ref: https://tools.ietf.org/html/rfc7234#section-4.2.1 - if maxage := GetTTLFromCacheControl("max-age", cacheControlValue); maxage > 0 { - ttl = maxage - } - - if smaxage := GetTTLFromCacheControl("s-maxage", cacheControlValue); smaxage > 0 { - ttl = smaxage - } - - if strings.Contains(cacheControlValue, "no-cache") || strings.Contains(cacheControlValue, "no-store") { - ttl = 0 + cacheControlTTL := ttlFromCacheControlChain(cacheControlValue) + if cacheControlTTL != nil { + ttl = *cacheControlTTL } } diff --git a/utils/ttl_test.go b/utils/ttl/ttl_test.go similarity index 67% rename from utils/ttl_test.go rename to utils/ttl/ttl_test.go index e1302046..c8e10ec0 100644 --- a/utils/ttl_test.go +++ b/utils/ttl/ttl_test.go @@ -1,6 +1,6 @@ -// +build unit +// +build all unit -package utils_test +package ttl_test // __ // .-----.-----.______.-----.----.-----.--.--.--.--.______.----.---.-.----| |--.-----. @@ -16,50 +16,50 @@ import ( "testing" "time" - "github.com/fabiocicerchia/go-proxy-cache/utils" + "github.com/fabiocicerchia/go-proxy-cache/utils/ttl" "github.com/stretchr/testify/assert" ) func TestGetTTLFromCacheControlWithMaxage(t *testing.T) { - value := utils.GetTTLFromCacheControl("max-age", `public, max-age=3600, s-maxage=86400`) + value := ttl.GetTTLFromCacheControl("max-age", `public, max-age=3600, s-maxage=86400`) assert.Equal(t, 3600*time.Second, value) - value = utils.GetTTLFromCacheControl("max-age", `public,max-age=3600,s-maxage=86400`) + value = ttl.GetTTLFromCacheControl("max-age", `public,max-age=3600,s-maxage=86400`) assert.Equal(t, 3600*time.Second, value) - value = utils.GetTTLFromCacheControl("max-age", `public, s-maxage=86400, max-age=3600`) + value = ttl.GetTTLFromCacheControl("max-age", `public, s-maxage=86400, max-age=3600`) assert.Equal(t, 3600*time.Second, value) - value = utils.GetTTLFromCacheControl("max-age", `public,s-maxage=86400,max-age=3600`) + value = ttl.GetTTLFromCacheControl("max-age", `public,s-maxage=86400,max-age=3600`) assert.Equal(t, 3600*time.Second, value) - value = utils.GetTTLFromCacheControl("max-age", `no-cache, max-age=0`) + value = ttl.GetTTLFromCacheControl("max-age", `no-cache, max-age=0`) assert.Equal(t, 0*time.Second, value) } func TestGetTTLFromCacheControlWithSmaxage(t *testing.T) { - value := utils.GetTTLFromCacheControl("s-maxage", `public, max-age=3600, s-maxage=86400`) + value := ttl.GetTTLFromCacheControl("s-maxage", `public, max-age=3600, s-maxage=86400`) assert.Equal(t, 86400*time.Second, value) - value = utils.GetTTLFromCacheControl("s-maxage", `public,max-age=3600,s-maxage=86400`) + value = ttl.GetTTLFromCacheControl("s-maxage", `public,max-age=3600,s-maxage=86400`) assert.Equal(t, 86400*time.Second, value) - value = utils.GetTTLFromCacheControl("s-maxage", `public, s-maxage=86400, max-age=3600`) + value = ttl.GetTTLFromCacheControl("s-maxage", `public, s-maxage=86400, max-age=3600`) assert.Equal(t, 86400*time.Second, value) - value = utils.GetTTLFromCacheControl("s-maxage", `public,s-maxage=86400,max-age=3600`) + value = ttl.GetTTLFromCacheControl("s-maxage", `public,s-maxage=86400,max-age=3600`) assert.Equal(t, 86400*time.Second, value) - value = utils.GetTTLFromCacheControl("s-maxage", `public,max-age=3600`) + value = ttl.GetTTLFromCacheControl("s-maxage", `public,max-age=3600`) assert.Equal(t, 0*time.Second, value) - value = utils.GetTTLFromCacheControl("s-maxage", `no-cache, max-age=0`) + value = ttl.GetTTLFromCacheControl("s-maxage", `no-cache, max-age=0`) assert.Equal(t, 0*time.Second, value) } func TestGetTTLWhenNotSet(t *testing.T) { headers := http.Header{} - value := utils.GetTTL(headers, 1) + value := ttl.GetTTL(headers, 1) assert.Equal(t, 1*time.Second, value) } @@ -67,7 +67,7 @@ func TestGetTTLWhenSetCacheControl(t *testing.T) { headers := http.Header{ "Cache-Control": []string{"public, max-age=3600, s-maxage=86400"}, } - value := utils.GetTTL(headers, 1) + value := ttl.GetTTL(headers, 1) assert.Equal(t, 86400*time.Second, value) } @@ -75,7 +75,7 @@ func TestGetTTLWhenCacheControlNoCache(t *testing.T) { headers := http.Header{ "Cache-Control": []string{"private, no-cache, max-age=3600"}, } - value := utils.GetTTL(headers, 1) + value := ttl.GetTTL(headers, 1) assert.Equal(t, 0*time.Second, value) } @@ -83,7 +83,7 @@ func TestGetTTLWhenCacheControlNoStore(t *testing.T) { headers := http.Header{ "Cache-Control": []string{"private, no-store, max-age=3600"}, } - value := utils.GetTTL(headers, 1) + value := ttl.GetTTL(headers, 1) assert.Equal(t, 0*time.Second, value) } @@ -94,7 +94,7 @@ func TestGetTTLWhenSetExpires(t *testing.T) { headers := http.Header{ "Expires": []string{expires}, } - value := utils.GetTTL(headers, 1) + value := ttl.GetTTL(headers, 1) assert.Less(t, float64(59), value.Seconds()) assert.Greater(t, float64(60), value.Seconds()) } @@ -107,6 +107,6 @@ func TestGetTTLWhenSetCacheControlAndExpires(t *testing.T) { "Cache-Control": []string{"public, max-age=3600, s-maxage=86400"}, "Expires": []string{expires}, } - value := utils.GetTTL(headers, 1) + value := ttl.GetTTL(headers, 1) assert.Equal(t, 86400*time.Second, value) } diff --git a/utils/utils_test.go b/utils/utils_test.go index fdfca59b..9391d323 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -1,4 +1,4 @@ -// +build unit +// +build all unit package utils_test