From 3200dd4931b4b2df4fed993e67c836853e989a3a Mon Sep 17 00:00:00 2001 From: Howard Wu Date: Thu, 27 Jun 2024 10:28:24 +1200 Subject: [PATCH] chore: go-resuable, and updated a dependency --- .github/workflows/build.yml | 79 ++--- .../data_holdings_test.go | 3 +- cmd/fdsn-holdings-consumer/log.go | 3 +- cmd/fdsn-quake-consumer/log.go | 3 +- cmd/fdsn-ws/data_holdings.go | 3 +- cmd/fdsn-ws/data_holdings_test.go | 7 +- cmd/fdsn-ws/fdsn_dataselect.go | 9 +- cmd/fdsn-ws/fdsn_event.go | 2 +- cmd/fdsn-ws/fdsn_station.go | 41 +-- cmd/fdsn-ws/fdsn_station_test.go | 33 ++- cmd/fdsn-ws/fdsn_station_type.go | 2 + cmd/fdsn-ws/log.go | 5 +- cmd/fdsn-ws/routes.go | 2 +- cmd/fdsn-ws/routes_integration_test.go | 5 +- cmd/fdsn-ws/sc3ml.go | 3 +- cmd/fdsn-ws/server.go | 27 +- cmd/fdsn-ws/server_test.go | 10 + cmd/s3-notify/s3-notify.go | 3 + go.mod | 3 +- go.sum | 6 +- internal/fdsn/dataselect.go | 30 +- internal/holdings/holdings_test.go | 3 +- internal/valid/valid_test.go | 3 +- .../github.com/gorilla/schema/.editorconfig | 20 ++ vendor/github.com/gorilla/schema/.gitignore | 1 + vendor/github.com/gorilla/schema/LICENSE | 2 +- vendor/github.com/gorilla/schema/Makefile | 34 +++ vendor/github.com/gorilla/schema/README.md | 39 ++- vendor/github.com/gorilla/schema/cache.go | 28 +- vendor/github.com/gorilla/schema/converter.go | 77 +++++ vendor/github.com/gorilla/schema/decoder.go | 85 +++++- vendor/github.com/gorilla/schema/encoder.go | 17 +- vendor/github.com/joho/godotenv/.gitignore | 1 + vendor/github.com/joho/godotenv/LICENCE | 23 ++ vendor/github.com/joho/godotenv/README.md | 202 +++++++++++++ vendor/github.com/joho/godotenv/godotenv.go | 228 +++++++++++++++ vendor/github.com/joho/godotenv/parser.go | 271 ++++++++++++++++++ vendor/modules.txt | 7 +- 38 files changed, 1168 insertions(+), 152 deletions(-) create mode 100644 vendor/github.com/gorilla/schema/.editorconfig create mode 100644 vendor/github.com/gorilla/schema/.gitignore create mode 100644 vendor/github.com/gorilla/schema/Makefile create mode 100644 vendor/github.com/joho/godotenv/.gitignore create mode 100644 vendor/github.com/joho/godotenv/LICENCE create mode 100644 vendor/github.com/joho/godotenv/README.md create mode 100644 vendor/github.com/joho/godotenv/godotenv.go create mode 100644 vendor/github.com/joho/godotenv/parser.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f6c809e6..db975132 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,8 +32,35 @@ jobs: - name: check output run: | jq . <<< '${{ steps.set.outputs.matrix }}' + build-app: + uses: GeoNet/Actions/.github/workflows/reusable-go-apps.yml@main + with: + buildSetup: | + sudo apt-get -yq update + sudo apt-get install -y xsltproc + testSetup: | + sudo apt-get -yq update + sudo apt-get install -y xsltproc + docker \ + run -d \ + -p 5432:5432 \ + -e POSTGRES_PASSWORD=test \ + -e POSTGRES_USER=fdsn_w \ + -e POSTGRES_DB=fdsn \ + --name postgres \ + docker.io/postgis/postgis:15-3.3-alpine + echo "Waiting until Postgres is ready..." + until nc -zv -w 1 127.0.0.1 5432; do + sleep 1s + done + sleep 5s + docker logs postgres + echo "Postgres is ready" + psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/drop-create.ddl + psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/create-users.ddl + goTestExtraArgs: -p 1 build: - needs: prepare + needs: [prepare, build-app] strategy: matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} uses: GeoNet/Actions/.github/workflows/reusable-docker-build.yml@main @@ -66,52 +93,4 @@ jobs: aws-region: ap-southeast-2 aws-role-arn-to-assume: arn:aws:iam::862640294325:role/github-actions-geonet-ecr-push aws-role-duration-seconds: "3600" - go-build: - if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} - uses: GeoNet/Actions/.github/workflows/reusable-go-build-smoke-test.yml@main - with: - paths: ${{ inputs.paths }} - gofmt: - if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} - uses: GeoNet/Actions/.github/workflows/reusable-gofmt.yml@main - golangci-lint: - if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} - uses: GeoNet/Actions/.github/workflows/reusable-golangci-lint.yml@main - go-vet: - if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} - uses: GeoNet/Actions/.github/workflows/reusable-go-vet.yml@main - go-test: - runs-on: ubuntu-latest - env: - AWS_REGION: ap-southeast-2 - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: go.mod - cache-dependency-path: go.sum - check-latest: true - - name: setup - run: | - sudo apt-get -yq update - sudo apt-get install -y xsltproc - docker \ - run -d \ - -p 5432:5432 \ - -e POSTGRES_PASSWORD=test \ - -e POSTGRES_USER=fdsn_w \ - -e POSTGRES_DB=fdsn \ - --name postgres \ - docker.io/postgis/postgis:15-3.3-alpine - echo "Waiting until Postgres is ready..." - until nc -zv -w 1 127.0.0.1 5432; do - sleep 1s - done - sleep 5s - docker logs postgres - echo "Postgres is ready" - psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/drop-create.ddl - psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/create-users.ddl - - name: test - run: | - ./all.sh + diff --git a/cmd/fdsn-holdings-consumer/data_holdings_test.go b/cmd/fdsn-holdings-consumer/data_holdings_test.go index b781db7e..c5b145fa 100644 --- a/cmd/fdsn-holdings-consumer/data_holdings_test.go +++ b/cmd/fdsn-holdings-consumer/data_holdings_test.go @@ -2,9 +2,10 @@ package main import ( "database/sql" - "github.com/GeoNet/fdsn/internal/holdings" "testing" "time" + + "github.com/GeoNet/fdsn/internal/holdings" ) func TestSaveHoldings(t *testing.T) { diff --git a/cmd/fdsn-holdings-consumer/log.go b/cmd/fdsn-holdings-consumer/log.go index 97301950..8f831791 100644 --- a/cmd/fdsn-holdings-consumer/log.go +++ b/cmd/fdsn-holdings-consumer/log.go @@ -1,9 +1,10 @@ package main import ( - "github.com/GeoNet/kit/metrics" "log" "os" + + "github.com/GeoNet/kit/metrics" ) var Prefix string diff --git a/cmd/fdsn-quake-consumer/log.go b/cmd/fdsn-quake-consumer/log.go index 97301950..8f831791 100644 --- a/cmd/fdsn-quake-consumer/log.go +++ b/cmd/fdsn-quake-consumer/log.go @@ -1,9 +1,10 @@ package main import ( - "github.com/GeoNet/kit/metrics" "log" "os" + + "github.com/GeoNet/kit/metrics" ) var Prefix string diff --git a/cmd/fdsn-ws/data_holdings.go b/cmd/fdsn-ws/data_holdings.go index 5b6d03f9..82a1221a 100644 --- a/cmd/fdsn-ws/data_holdings.go +++ b/cmd/fdsn-ws/data_holdings.go @@ -2,8 +2,9 @@ package main import ( "database/sql" - "github.com/GeoNet/fdsn/internal/fdsn" "time" + + "github.com/GeoNet/fdsn/internal/fdsn" ) type metric struct { diff --git a/cmd/fdsn-ws/data_holdings_test.go b/cmd/fdsn-ws/data_holdings_test.go index 690b3770..b661b7cd 100644 --- a/cmd/fdsn-ws/data_holdings_test.go +++ b/cmd/fdsn-ws/data_holdings_test.go @@ -1,12 +1,13 @@ package main import ( - "github.com/GeoNet/fdsn/internal/fdsn" - "github.com/GeoNet/fdsn/internal/holdings" - "github.com/lib/pq" "log" "testing" "time" + + "github.com/GeoNet/fdsn/internal/fdsn" + "github.com/GeoNet/fdsn/internal/holdings" + "github.com/lib/pq" ) // http://www.postgresql.org/docs/9.4/static/errcodes-appendix.html diff --git a/cmd/fdsn-ws/fdsn_dataselect.go b/cmd/fdsn-ws/fdsn_dataselect.go index 07bd2d53..52489375 100644 --- a/cmd/fdsn-ws/fdsn_dataselect.go +++ b/cmd/fdsn-ws/fdsn_dataselect.go @@ -8,9 +8,7 @@ import ( "log" "net/http" "os" - "reflect" "regexp" - "strings" "text/template" "time" @@ -41,12 +39,7 @@ type dataSelect struct { keys []string } -func init() { - // Handle comma separated parameters (eg: net, sta, loc, cha, etc) - decoder.RegisterConverter([]string{}, func(input string) reflect.Value { - return reflect.ValueOf(strings.Split(input, ",")) - }) - +func initDataselectTemplate() { var err error var b bytes.Buffer diff --git a/cmd/fdsn-ws/fdsn_event.go b/cmd/fdsn-ws/fdsn_event.go index 19bbcbde..f13f4153 100644 --- a/cmd/fdsn-ws/fdsn_event.go +++ b/cmd/fdsn-ws/fdsn_event.go @@ -95,7 +95,7 @@ var validEventTypes = strings.Split( ", ", ","), ",") // remove spaces after comma -func init() { +func initEventTemplate() { var err error var b bytes.Buffer diff --git a/cmd/fdsn-ws/fdsn_station.go b/cmd/fdsn-ws/fdsn_station.go index a86d3869..79dc83d5 100644 --- a/cmd/fdsn-ws/fdsn_station.go +++ b/cmd/fdsn-ws/fdsn_station.go @@ -106,13 +106,13 @@ var ( fdsnStationWadlFile []byte fdsnStationIndex []byte fdsnStations fdsnStationObj - emptyDateTime time.Time + emptyDateTime = time.Date(9999, 1, 1, 0, 0, 0, 0, time.UTC) errNotModified = fmt.Errorf("Not modified.") s3Bucket string s3Meta string ) -func init() { +func initStationTemplate() { var err error var b bytes.Buffer @@ -130,9 +130,10 @@ func init() { if err != nil { log.Printf("error reading assets/fdsn-ws-station.html: %s", err.Error()) } +} - emptyDateTime = time.Date(9999, 1, 1, 0, 0, 0, 0, time.UTC) - +func initStationXML() { + var err error s3Bucket = os.Getenv("STATION_XML_BUCKET") s3Meta = os.Getenv("STATION_XML_META_KEY") @@ -193,7 +194,7 @@ func parseStationV1Post(body string) ([]fdsnStationV1Search, error) { case "format": format = strings.TrimSpace(tokens[1]) if format != "xml" && format != "text" { - return ret, errors.New("Invalid format") + return ret, errors.New("invalid format") } } } else if tokens := strings.Fields(line); len(tokens) == 6 { @@ -220,12 +221,12 @@ func parseStationV1Post(body string) ([]fdsnStationV1Search, error) { } ret = append(ret, p) } else { - return ret, fmt.Errorf("Invalid query format (POST).") + return ret, fmt.Errorf("invalid query format (POST)") } } if level == "response" && format == "text" { - return []fdsnStationV1Search{}, fmt.Errorf("Text formats are only supported when level is net|sta|cha.") + return []fdsnStationV1Search{}, fmt.Errorf("text formats are only supported when level is net|sta|cha") } return ret, nil @@ -269,7 +270,7 @@ func parseStationV1(v url.Values) (fdsnStationV1Search, error) { for key, val := range v { if len(val[0]) == 0 { - return fdsnStationV1Search{}, fmt.Errorf("Invalid %s value", key) + return fdsnStationV1Search{}, fmt.Errorf("invalid %s value", key) } } @@ -280,11 +281,11 @@ func parseStationV1(v url.Values) (fdsnStationV1Search, error) { // Only xml and text is allowed. if p.Format != "xml" && p.Format != "text" { - return fdsnStationV1Search{}, fmt.Errorf("Invalid format.") + return fdsnStationV1Search{}, fmt.Errorf("invalid format") } if p.Level == "response" && p.Format == "text" { - return fdsnStationV1Search{}, fmt.Errorf("Text formats are only supported when level is net|sta|cha.") + return fdsnStationV1Search{}, fmt.Errorf("text formats are only supported when level is net|sta|cha") } count := 0 @@ -303,7 +304,7 @@ func parseStationV1(v url.Values) (fdsnStationV1Search, error) { p.StartTime = p.StartBefore } if count > 1 { - return fdsnStationV1Search{}, fmt.Errorf("Only one of 'starttime', 'startafter', and 'startbefore' is allowed.") + return fdsnStationV1Search{}, fmt.Errorf("only one of 'starttime', 'startafter', and 'startbefore' is allowed") } count = 0 @@ -325,23 +326,23 @@ func parseStationV1(v url.Values) (fdsnStationV1Search, error) { } if count > 1 { - return fdsnStationV1Search{}, fmt.Errorf("Only one of 'endtime', 'endafter', and 'endbefore' is allowed.") + return fdsnStationV1Search{}, fmt.Errorf("only one of 'endtime', 'endafter', and 'endbefore' is allowed") } if p.IncludeAvailability { - return fdsnStationV1Search{}, errors.New("include availability is not supported.") + return fdsnStationV1Search{}, errors.New("include availability is not supported") } if !p.IncludeRestricted { - return fdsnStationV1Search{}, errors.New("exclude restricted is not supported.") + return fdsnStationV1Search{}, errors.New("exclude restricted is not supported") } if p.MatchTimeSeries { - return fdsnStationV1Search{}, errors.New("match time series is not supported.") + return fdsnStationV1Search{}, errors.New("match time series is not supported") } if p.NoData != 204 && p.NoData != 404 { - return fdsnStationV1Search{}, errors.New("nodata must be 204 or 404.") + return fdsnStationV1Search{}, errors.New("nodata must be 204 or 404") } ne, err := fdsn.GenRegex(p.Network, false, false) @@ -397,7 +398,7 @@ func parseStationV1(v url.Values) (fdsnStationV1Search, error) { // Now validate longitude, latitude, and radius if p.Longitude != math.MaxFloat64 || p.Latitude != math.MaxFloat64 { if p.Longitude == math.MaxFloat64 || p.Latitude == math.MaxFloat64 { - err = fmt.Errorf("parameter latitude and longitude must both present.") + err = fmt.Errorf("parameter latitude and longitude must both present") return s, err } @@ -412,17 +413,17 @@ func parseStationV1(v url.Values) (fdsnStationV1Search, error) { } if p.MaxRadius < 0 || p.MaxRadius > 180.0 { - err = fmt.Errorf("invalid maxradius value.") + err = fmt.Errorf("invalid maxradius value") return s, err } if p.MinRadius < 0 || p.MinRadius > 180.0 { - err = fmt.Errorf("invalid minradius value.") + err = fmt.Errorf("invalid minradius value") return s, err } if p.MinRadius > p.MaxRadius { - err = fmt.Errorf("minradius or maxradius range error.") + err = fmt.Errorf("minradius or maxradius range error") return s, err } } diff --git a/cmd/fdsn-ws/fdsn_station_test.go b/cmd/fdsn-ws/fdsn_station_test.go index 1b36a502..1c91eed0 100644 --- a/cmd/fdsn-ws/fdsn_station_test.go +++ b/cmd/fdsn-ws/fdsn_station_test.go @@ -6,19 +6,19 @@ import ( "strings" "testing" + _ "github.com/GeoNet/fdsn/internal/fdsn" wt "github.com/GeoNet/kit/weft/wefttest" ) // NOTE: To run the test, please export : // STATION_XML_META_KEY=fdsn-station-test.xml -func init() { -} - func TestStationFilter(t *testing.T) { var e fdsnStationV1Search var err error + setup(t) + defer teardown() // Filter test var v url.Values = make(map[string][]string) @@ -207,13 +207,16 @@ Testdata timeline Station 1: 2007-05-20T23 2011-03-06T22 2011-06-20T04 - |------3 cha----| - |-----3 cha----| - |-----3 cha-----------> + + |------3 cha----| + |-----3 cha----| + |-----3 cha-----------> + Station 2: - 2010-03-11T21 2012-01-19T22 - |-----3 cha----| - |-----3 cha-------> + + 2010-03-11T21 2012-01-19T22 + |-----3 cha----| + |-----3 cha-------> */ func TestStartEnd(t *testing.T) { var e fdsnStationV1Search @@ -408,10 +411,11 @@ NZ|ARAZ|-38.627690|176.120060|420.000000|Aratiatia Landcorp Farm|2007-05-20T23:0 } // To profiling, you'll have to use full fdsn-station xml as data source: -// 1. Put full fdsn-station.xml in etc/. -// 2. export FDSN_STATION_XML_META_KEY=fdsn-station.xml -// 3. Run `go test -bench=StationQuery -benchmem -run=^$`. -// Note: You must specify -run=^$ to skip test functions since you're not using test fdsn-station xml. +// 1. Put full fdsn-station.xml in etc/. +// 2. export FDSN_STATION_XML_META_KEY=fdsn-station.xml +// 3. Run `go test -bench=StationQuery -benchmem -run=^$`. +// Note: You must specify -run=^$ to skip test functions since you're not using test fdsn-station xml. +// // Currently the benchmark result for my MacBookPro 2017 is: // BenchmarkStationQuery/post-4 20000 74472 ns/op 78376 B/op 706 allocs/op func BenchmarkStationQuery(b *testing.B) { @@ -585,6 +589,8 @@ func TestDoFilter(t *testing.T) { var query url.Values var hasValue bool + setup(t) + defer teardown() // // basic case // @@ -747,7 +753,6 @@ func testCase(c *FDSNStationXML, query url.Values) (bool, error) { if e, err = parseStationV1(query); err != nil { return false, err } - return c.doFilter([]fdsnStationV1Search{e}), nil } diff --git a/cmd/fdsn-ws/fdsn_station_type.go b/cmd/fdsn-ws/fdsn_station_type.go index cae58bf1..f46cefe9 100644 --- a/cmd/fdsn-ws/fdsn_station_type.go +++ b/cmd/fdsn-ws/fdsn_station_type.go @@ -247,6 +247,7 @@ type NetworkType struct { } // May be one of NOMINAL, CALCULATED +// //nolint:deadcode,unused // Struct based on FDSN spec so keep it type NominalType string @@ -364,6 +365,7 @@ type SampleRateType struct { } // A time value in seconds. +// //nolint:deadcode,unused // Struct based on FDSN spec so keep it type SecondType struct { Value float64 `xml:",chardata"` diff --git a/cmd/fdsn-ws/log.go b/cmd/fdsn-ws/log.go index 1cd0696f..cdf56ba7 100644 --- a/cmd/fdsn-ws/log.go +++ b/cmd/fdsn-ws/log.go @@ -1,10 +1,11 @@ package main import ( - "github.com/GeoNet/kit/metrics" - "github.com/GeoNet/kit/weft" "log" "os" + + "github.com/GeoNet/kit/metrics" + "github.com/GeoNet/kit/weft" ) var Prefix string diff --git a/cmd/fdsn-ws/routes.go b/cmd/fdsn-ws/routes.go index a6e65f61..76b276ae 100644 --- a/cmd/fdsn-ws/routes.go +++ b/cmd/fdsn-ws/routes.go @@ -12,7 +12,7 @@ import ( var mux *http.ServeMux -func init() { +func initRoutes() { mux = http.NewServeMux() mux.HandleFunc("/", weft.MakeHandler(weft.NoMatch, weft.TextError)) diff --git a/cmd/fdsn-ws/routes_integration_test.go b/cmd/fdsn-ws/routes_integration_test.go index 36f7ffc8..786a255e 100644 --- a/cmd/fdsn-ws/routes_integration_test.go +++ b/cmd/fdsn-ws/routes_integration_test.go @@ -1,10 +1,12 @@ +//go:build integration // +build integration package main import ( - wt "github.com/GeoNet/kit/weft/wefttest" "testing" + + wt "github.com/GeoNet/kit/weft/wefttest" ) var routesIntegration = wt.Requests{ @@ -19,7 +21,6 @@ var routesIntegration = wt.Requests{ // Run using: // go test -tags integration -v -run TestRoutesIntegration func TestRoutesIntegration(t *testing.T) { - setup(t) setup(t) defer teardown() diff --git a/cmd/fdsn-ws/sc3ml.go b/cmd/fdsn-ws/sc3ml.go index 40db1034..1313f222 100644 --- a/cmd/fdsn-ws/sc3ml.go +++ b/cmd/fdsn-ws/sc3ml.go @@ -2,9 +2,10 @@ package main import ( "bytes" + "net/http" + "github.com/GeoNet/fdsn/internal/valid" "github.com/GeoNet/kit/weft" - "net/http" ) func s3ml(r *http.Request, h http.Header, b *bytes.Buffer) error { diff --git a/cmd/fdsn-ws/server.go b/cmd/fdsn-ws/server.go index 3ffa8099..f3311e36 100644 --- a/cmd/fdsn-ws/server.go +++ b/cmd/fdsn-ws/server.go @@ -5,6 +5,8 @@ import ( "log" "net/http" "os" + "reflect" + "strings" "time" "github.com/GeoNet/kit/cfg" @@ -13,19 +15,24 @@ import ( ) var ( - db *sql.DB - decoder = schema.NewDecoder() // decoder for URL queries. - S3_BUCKET string // the S3 bucket storing the miniseed files used by dataselect - LOG_EXTRA bool // Whether POST body is logged. - zeroDateTime time.Time + db *sql.DB + decoder = newDecoder() // decoder for URL queries. + S3_BUCKET string // the S3 bucket storing the miniseed files used by dataselect + LOG_EXTRA bool // Whether POST body is logged. ) var stationVersion = "1.1" var eventVersion = "1.2" var dataselectVersion = "1.1" +var zeroDateTime = time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC) -func init() { - zeroDateTime = time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC) +func newDecoder() *schema.Decoder { + decoder := schema.NewDecoder() + // Handle comma separated parameters (eg: net, sta, loc, cha, etc) + decoder.RegisterConverter([]string{}, func(input string) reflect.Value { + return reflect.ValueOf(strings.Split(input, ",")) + }) + return decoder } func main() { @@ -60,6 +67,12 @@ func main() { log.Println("ERROR: problem pinging DB - is it up and contactable? 500s will be served") } + initDataselectTemplate() + initEventTemplate() + initStationTemplate() + initStationXML() + initRoutes() + setupStationXMLUpdater() log.Println("starting server") diff --git a/cmd/fdsn-ws/server_test.go b/cmd/fdsn-ws/server_test.go index aa0aac3c..c282a307 100644 --- a/cmd/fdsn-ws/server_test.go +++ b/cmd/fdsn-ws/server_test.go @@ -7,6 +7,8 @@ import ( "net/http/httptest" "os" "testing" + + "github.com/joho/godotenv" ) var ts *httptest.Server @@ -14,6 +16,14 @@ var ts *httptest.Server func setup(t *testing.T) { var err error + err = godotenv.Load("env.list") + if err != nil { + t.Fatal(err) + } + + initStationXML() + initRoutes() + S3_BUCKET = os.Getenv("S3_BUCKET") // need a db write user for adding test data. diff --git a/cmd/s3-notify/s3-notify.go b/cmd/s3-notify/s3-notify.go index 22eae8fe..c4bc3bf4 100644 --- a/cmd/s3-notify/s3-notify.go +++ b/cmd/s3-notify/s3-notify.go @@ -24,7 +24,9 @@ func init() { flag.StringVar(&keyPrefix, "key-prefix", "", "Key prefix to search in the S3 bucket.") flag.StringVar(&sqsUrl, "sqs-url", "", "SQS queue url to send notifications to. Omit this parameter to show the list of matched keys only.") flag.Parse() +} +func initAWS() { var err error s3c, err := s3.NewWithMaxRetries(100) if err != nil { @@ -50,6 +52,7 @@ func main() { fmt.Println("Send to SQS:", sqsUrl) } + initAWS() keys, err := s3Client.ListAll(bucketName, keyPrefix) if err != nil { log.Fatalf("error listing S3 objects: %s", err) diff --git a/go.mod b/go.mod index 9a67bd03..9f21b413 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,8 @@ go 1.21 require ( github.com/GeoNet/kit v0.0.0-20240512234353-4d4493144f60 - github.com/gorilla/schema v1.2.0 + github.com/gorilla/schema v1.4.0 + github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.3 ) diff --git a/go.sum b/go.sum index 1c2e1e7e..d7e44815 100644 --- a/go.sum +++ b/go.sum @@ -42,12 +42,14 @@ github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gorilla/schema v1.2.0 h1:YufUaxZYCKGFuAq3c96BOhjgd5nmXiOY9NGzF247Tsc= -github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= +github.com/gorilla/schema v1.4.0 h1:l2N+lRTJtev9SUhBtj6NmSxd/6+8LhvN0kV+H2Y8R9k= +github.com/gorilla/schema v1.4.0/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/fdsn/dataselect.go b/internal/fdsn/dataselect.go index 6b24ad44..b5dac85c 100644 --- a/internal/fdsn/dataselect.go +++ b/internal/fdsn/dataselect.go @@ -5,7 +5,6 @@ import ( "bufio" "errors" "fmt" - "github.com/gorilla/schema" "io" "net/url" "reflect" @@ -13,6 +12,8 @@ import ( "strconv" "strings" "time" + + "github.com/gorilla/schema" ) var decoder = schema.NewDecoder() @@ -71,9 +72,10 @@ func init() { /* parses the time in text as per the FDSN spec. Pads text for parsing with time.RFC3339Nano. Accepted formats are (UTC): - YYYY-MM-DDTHH:MM:SS.ssssss - YYYY-MM-DDTHH:MM:SS - YYYY-MM-DD + + YYYY-MM-DDTHH:MM:SS.ssssss + YYYY-MM-DDTHH:MM:SS + YYYY-MM-DD Implements the encoding.TextUnmarshaler interface. */ @@ -119,7 +121,7 @@ func ParseDataSelectPost(r io.Reader, d *[]DataSelect) error { } if noData != 204 && noData != 404 { - return errors.New("nodata must be 204 or 404.") + return errors.New("nodata must be 204 or 404") } } } @@ -186,7 +188,7 @@ func ParseDataSelectGet(v url.Values) (DataSelect, error) { return DataSelect{}, fmt.Errorf("\"%s\" is not supported", key) } if len(val[0]) == 0 { - return DataSelect{}, fmt.Errorf("Invalid %s value", key) + return DataSelect{}, fmt.Errorf("invalid %s value", key) } } @@ -196,15 +198,15 @@ func ParseDataSelectGet(v url.Values) (DataSelect, error) { } if e.Format != "miniseed" { - return DataSelect{}, fmt.Errorf("Only \"miniseed\" format is supported.") + return DataSelect{}, fmt.Errorf("only \"miniseed\" format is supported") } if e.LongestOnly { - return DataSelect{}, fmt.Errorf("Query for longest only is not supported.") + return DataSelect{}, fmt.Errorf("query for longest only is not supported") } if e.NoData != 204 && e.NoData != 404 { - return DataSelect{}, errors.New("nodata must be 204 or 404.") + return DataSelect{}, errors.New("nodata must be 204 or 404") } // Defaults: as per spec we need to include any valid files in the search so use wildcards and broad time range @@ -234,22 +236,22 @@ func ParseDataSelectGet(v url.Values) (DataSelect, error) { func (d *DataSelect) Regexp() (DataSearch, error) { ne, err := toPattern(d.Network, false) if err != nil { - return DataSearch{}, fmt.Errorf("Invalid network parameter: %s", err.Error()) + return DataSearch{}, fmt.Errorf("invalid network parameter: %s", err.Error()) } st, err := toPattern(d.Station, false) if err != nil { - return DataSearch{}, fmt.Errorf("Invalid station parameter: %s", err.Error()) + return DataSearch{}, fmt.Errorf("invalid station parameter: %s", err.Error()) } ch, err := toPattern(d.Channel, false) if err != nil { - return DataSearch{}, fmt.Errorf("Invalid channel parameter: %s", err.Error()) + return DataSearch{}, fmt.Errorf("invalid channel parameter: %s", err.Error()) } lo, err := toPattern(d.Location, true) if err != nil { - return DataSearch{}, fmt.Errorf("Invalid location parameter: %s", err.Error()) + return DataSearch{}, fmt.Errorf("invalid location parameter: %s", err.Error()) } return DataSearch{ @@ -290,7 +292,7 @@ func GenRegex(input []string, emptyDash bool, allowSpace bool) ([]string, error) matched = nslcReg.MatchString(s) } if !matched { - return nil, fmt.Errorf("Invalid parameter:'%s'", s) + return nil, fmt.Errorf("invalid parameter:'%s'", s) } var r string diff --git a/internal/holdings/holdings_test.go b/internal/holdings/holdings_test.go index 8896ca1d..7a255b95 100644 --- a/internal/holdings/holdings_test.go +++ b/internal/holdings/holdings_test.go @@ -1,11 +1,12 @@ package holdings_test import ( - "github.com/GeoNet/fdsn/internal/holdings" "os" "reflect" "testing" "time" + + "github.com/GeoNet/fdsn/internal/holdings" ) type result struct { diff --git a/internal/valid/valid_test.go b/internal/valid/valid_test.go index ab8135b1..3b3785f1 100644 --- a/internal/valid/valid_test.go +++ b/internal/valid/valid_test.go @@ -1,11 +1,12 @@ package valid_test import ( - "github.com/GeoNet/fdsn/internal/valid" "net/http" "runtime" "strconv" "testing" + + "github.com/GeoNet/fdsn/internal/valid" ) var bad = &valid.Error{Code: http.StatusBadRequest} diff --git a/vendor/github.com/gorilla/schema/.editorconfig b/vendor/github.com/gorilla/schema/.editorconfig new file mode 100644 index 00000000..c6b74c3e --- /dev/null +++ b/vendor/github.com/gorilla/schema/.editorconfig @@ -0,0 +1,20 @@ +; https://editorconfig.org/ + +root = true + +[*] +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[{Makefile,go.mod,go.sum,*.go,.gitmodules}] +indent_style = tab +indent_size = 4 + +[*.md] +indent_size = 4 +trim_trailing_whitespace = false + +eclint_indent_style = unset \ No newline at end of file diff --git a/vendor/github.com/gorilla/schema/.gitignore b/vendor/github.com/gorilla/schema/.gitignore new file mode 100644 index 00000000..84039fec --- /dev/null +++ b/vendor/github.com/gorilla/schema/.gitignore @@ -0,0 +1 @@ +coverage.coverprofile diff --git a/vendor/github.com/gorilla/schema/LICENSE b/vendor/github.com/gorilla/schema/LICENSE index 0e5fb872..bb9d80bc 100644 --- a/vendor/github.com/gorilla/schema/LICENSE +++ b/vendor/github.com/gorilla/schema/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012 Rodrigo Moraes. All rights reserved. +Copyright (c) 2023 The Gorilla Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/vendor/github.com/gorilla/schema/Makefile b/vendor/github.com/gorilla/schema/Makefile new file mode 100644 index 00000000..98f5ab75 --- /dev/null +++ b/vendor/github.com/gorilla/schema/Makefile @@ -0,0 +1,34 @@ +GO_LINT=$(shell which golangci-lint 2> /dev/null || echo '') +GO_LINT_URI=github.com/golangci/golangci-lint/cmd/golangci-lint@latest + +GO_SEC=$(shell which gosec 2> /dev/null || echo '') +GO_SEC_URI=github.com/securego/gosec/v2/cmd/gosec@latest + +GO_VULNCHECK=$(shell which govulncheck 2> /dev/null || echo '') +GO_VULNCHECK_URI=golang.org/x/vuln/cmd/govulncheck@latest + +.PHONY: golangci-lint +golangci-lint: + $(if $(GO_LINT), ,go install $(GO_LINT_URI)) + @echo "##### Running golangci-lint" + golangci-lint run -v + +.PHONY: gosec +gosec: + $(if $(GO_SEC), ,go install $(GO_SEC_URI)) + @echo "##### Running gosec" + gosec ./... + +.PHONY: govulncheck +govulncheck: + $(if $(GO_VULNCHECK), ,go install $(GO_VULNCHECK_URI)) + @echo "##### Running govulncheck" + govulncheck ./... + +.PHONY: verify +verify: golangci-lint gosec govulncheck + +.PHONY: test +test: + @echo "##### Running tests" + go test -race -cover -coverprofile=coverage.coverprofile -covermode=atomic -v ./... \ No newline at end of file diff --git a/vendor/github.com/gorilla/schema/README.md b/vendor/github.com/gorilla/schema/README.md index aefdd669..58786ba5 100644 --- a/vendor/github.com/gorilla/schema/README.md +++ b/vendor/github.com/gorilla/schema/README.md @@ -1,8 +1,12 @@ -schema -====== -[![GoDoc](https://godoc.org/github.com/gorilla/schema?status.svg)](https://godoc.org/github.com/gorilla/schema) [![Build Status](https://travis-ci.org/gorilla/schema.png?branch=master)](https://travis-ci.org/gorilla/schema) -[![Sourcegraph](https://sourcegraph.com/github.com/gorilla/schema/-/badge.svg)](https://sourcegraph.com/github.com/gorilla/schema?badge) +# gorilla/schema +![testing](https://github.com/gorilla/schema/actions/workflows/test.yml/badge.svg) +[![codecov](https://codecov.io/github/gorilla/schema/branch/main/graph/badge.svg)](https://codecov.io/github/gorilla/schema) +[![godoc](https://godoc.org/github.com/gorilla/schema?status.svg)](https://godoc.org/github.com/gorilla/schema) +[![sourcegraph](https://sourcegraph.com/github.com/gorilla/schema/-/badge.svg)](https://sourcegraph.com/github.com/gorilla/schema?badge) + + +![Gorilla Logo](https://github.com/gorilla/.github/assets/53367916/d92caabf-98e0-473e-bfbf-ab554ba435e5) Package gorilla/schema converts structs to and from form values. @@ -83,7 +87,32 @@ The supported field types in the struct are: Unsupported types are simply ignored, however custom types can be registered to be converted. -More examples are available on the Gorilla website: https://www.gorillatoolkit.org/pkg/schema +## Setting Defaults + +It is possible to set default values when encoding/decoding by using the `default` tag option. The value of `default` is applied when a field has a zero value, a pointer has a nil value, or a slice is empty. + +```go +type Person struct { + Phone string `schema:"phone,default:+123456"` // custom name + Age int `schema:"age,default:21"` + Admin bool `schema:"admin,default:false"` + Balance float64 `schema:"balance,default:10.0"` + Friends []string `schema:friends,default:john|bob` +} +``` + +The `default` tag option is supported for the following types: + +* bool +* float variants (float32, float64) +* int variants (int, int8, int16, int32, int64) +* uint variants (uint, uint8, uint16, uint32, uint64) +* string +* a slice of the above types. As shown in the example above, `|` should be used to separate between slice items. +* a pointer to one of the above types (pointer to slice and slice of pointers are not supported). + +> [!NOTE] +> Because primitive types like int, float, bool, unint and their variants have their default (or zero) values set by Golang, it is not possible to distinguish them from a provided value when decoding/encoding form values. In this case, the value provided by the `default` option tag will be always applied. For example, let's assume that the value submitted in the form for `balance` is `0.0` then the default of `10.0` will be applied, even if `0.0` is part of the form data for the `balance` field. In such cases, it is highly recommended to use pointers to allow schema to distinguish between when a form field has no provided value and when a form has a value equal to the corresponding default set by Golang for a particular type. If the type of the `Balance` field above is changed to `*float64`, then the zero value would be `nil`. In this case, if the form data value for `balance` is `0.0`, then the default will not be applied. ## License diff --git a/vendor/github.com/gorilla/schema/cache.go b/vendor/github.com/gorilla/schema/cache.go index 0746c120..065b8d6e 100644 --- a/vendor/github.com/gorilla/schema/cache.go +++ b/vendor/github.com/gorilla/schema/cache.go @@ -12,7 +12,7 @@ import ( "sync" ) -var invalidPath = errors.New("schema: invalid path") +var errInvalidPath = errors.New("schema: invalid path") // newCache returns a new cache. func newCache() *cache { @@ -53,13 +53,13 @@ func (c *cache) parsePath(p string, t reflect.Type) ([]pathPart, error) { keys := strings.Split(p, ".") for i := 0; i < len(keys); i++ { if t.Kind() != reflect.Struct { - return nil, invalidPath + return nil, errInvalidPath } if struc = c.get(t); struc == nil { - return nil, invalidPath + return nil, errInvalidPath } if field = struc.get(keys[i]); field == nil { - return nil, invalidPath + return nil, errInvalidPath } // Valid field. Append index. path = append(path, field.name) @@ -72,10 +72,10 @@ func (c *cache) parsePath(p string, t reflect.Type) ([]pathPart, error) { // So checking i+2 is not necessary anymore. i++ if i+1 > len(keys) { - return nil, invalidPath + return nil, errInvalidPath } if index64, err = strconv.ParseInt(keys[i], 10, 0); err != nil { - return nil, invalidPath + return nil, errInvalidPath } parts = append(parts, pathPart{ path: path, @@ -197,6 +197,7 @@ func (c *cache) createField(field reflect.StructField, parentAlias string) *fiel isSliceOfStructs: isSlice && isStruct, isAnonymous: field.Anonymous, isRequired: options.Contains("required"), + defaultValue: options.getDefaultOptionValue(), } } @@ -246,8 +247,9 @@ type fieldInfo struct { // isSliceOfStructs indicates if the field type is a slice of structs. isSliceOfStructs bool // isAnonymous indicates whether the field is embedded in the struct. - isAnonymous bool - isRequired bool + isAnonymous bool + isRequired bool + defaultValue string } func (f *fieldInfo) paths(prefix string) []string { @@ -303,3 +305,13 @@ func (o tagOptions) Contains(option string) bool { } return false } + +func (o tagOptions) getDefaultOptionValue() string { + for _, s := range o { + if strings.HasPrefix(s, "default:") { + return strings.Split(s, ":")[1] + } + } + + return "" +} diff --git a/vendor/github.com/gorilla/schema/converter.go b/vendor/github.com/gorilla/schema/converter.go index 4f2116a1..4bae6df9 100644 --- a/vendor/github.com/gorilla/schema/converter.go +++ b/vendor/github.com/gorilla/schema/converter.go @@ -143,3 +143,80 @@ func convertUint64(value string) reflect.Value { } return invalidValue } + +func convertPointer(k reflect.Kind, value string) reflect.Value { + switch k { + case boolType: + if v := convertBool(value); v.IsValid() { + converted := v.Bool() + return reflect.ValueOf(&converted) + } + case float32Type: + if v := convertFloat32(value); v.IsValid() { + converted := float32(v.Float()) + return reflect.ValueOf(&converted) + } + case float64Type: + if v := convertFloat64(value); v.IsValid() { + converted := float64(v.Float()) + return reflect.ValueOf(&converted) + } + case intType: + if v := convertInt(value); v.IsValid() { + converted := int(v.Int()) + return reflect.ValueOf(&converted) + } + case int8Type: + if v := convertInt8(value); v.IsValid() { + converted := int8(v.Int()) + return reflect.ValueOf(&converted) + } + case int16Type: + if v := convertInt16(value); v.IsValid() { + converted := int16(v.Int()) + return reflect.ValueOf(&converted) + } + case int32Type: + if v := convertInt32(value); v.IsValid() { + converted := int32(v.Int()) + return reflect.ValueOf(&converted) + } + case int64Type: + if v := convertInt64(value); v.IsValid() { + converted := int64(v.Int()) + return reflect.ValueOf(&converted) + } + case stringType: + if v := convertString(value); v.IsValid() { + converted := v.String() + return reflect.ValueOf(&converted) + } + case uintType: + if v := convertUint(value); v.IsValid() { + converted := uint(v.Uint()) + return reflect.ValueOf(&converted) + } + case uint8Type: + if v := convertUint8(value); v.IsValid() { + converted := uint8(v.Uint()) + return reflect.ValueOf(&converted) + } + case uint16Type: + if v := convertUint16(value); v.IsValid() { + converted := uint16(v.Uint()) + return reflect.ValueOf(&converted) + } + case uint32Type: + if v := convertUint32(value); v.IsValid() { + converted := uint32(v.Uint()) + return reflect.ValueOf(&converted) + } + case uint64Type: + if v := convertUint64(value); v.IsValid() { + converted := uint64(v.Uint()) + return reflect.ValueOf(&converted) + } + } + + return invalidValue +} diff --git a/vendor/github.com/gorilla/schema/decoder.go b/vendor/github.com/gorilla/schema/decoder.go index 025e438b..ed856419 100644 --- a/vendor/github.com/gorilla/schema/decoder.go +++ b/vendor/github.com/gorilla/schema/decoder.go @@ -84,6 +84,7 @@ func (d *Decoder) Decode(dst interface{}, src map[string][]string) error { errors[path] = UnknownKeyError{Key: path} } } + errors.merge(d.setDefaults(t, v)) errors.merge(d.checkRequired(t, src)) if len(errors) > 0 { return errors @@ -91,6 +92,88 @@ func (d *Decoder) Decode(dst interface{}, src map[string][]string) error { return nil } +// setDefaults sets the default values when the `default` tag is specified, +// default is supported on basic/primitive types and their pointers, +// nested structs can also have default tags +func (d *Decoder) setDefaults(t reflect.Type, v reflect.Value) MultiError { + struc := d.cache.get(t) + if struc == nil { + // unexpect, cache.get never return nil + return MultiError{"default-" + t.Name(): errors.New("cache fail")} + } + + errs := MultiError{} + + if v.Type().Kind() == reflect.Struct { + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + if field.Type().Kind() == reflect.Ptr && field.IsNil() && v.Type().Field(i).Anonymous { + field.Set(reflect.New(field.Type().Elem())) + } + } + } + + for _, f := range struc.fields { + vCurrent := v.FieldByName(f.name) + + if vCurrent.Type().Kind() == reflect.Struct && f.defaultValue == "" { + errs.merge(d.setDefaults(vCurrent.Type(), vCurrent)) + } else if isPointerToStruct(vCurrent) && f.defaultValue == "" { + errs.merge(d.setDefaults(vCurrent.Elem().Type(), vCurrent.Elem())) + } + + if f.defaultValue != "" && f.isRequired { + errs.merge(MultiError{"default-" + f.name: errors.New("required fields cannot have a default value")}) + } else if f.defaultValue != "" && vCurrent.IsZero() && !f.isRequired { + if f.typ.Kind() == reflect.Struct { + errs.merge(MultiError{"default-" + f.name: errors.New("default option is supported only on: bool, float variants, string, unit variants types or their corresponding pointers or slices")}) + } else if f.typ.Kind() == reflect.Slice { + vals := strings.Split(f.defaultValue, "|") + + // check if slice has one of the supported types for defaults + if _, ok := builtinConverters[f.typ.Elem().Kind()]; !ok { + errs.merge(MultiError{"default-" + f.name: errors.New("default option is supported only on: bool, float variants, string, unit variants types or their corresponding pointers or slices")}) + continue + } + + defaultSlice := reflect.MakeSlice(f.typ, 0, cap(vals)) + for _, val := range vals { + // this check is to handle if the wrong value is provided + convertedVal := builtinConverters[f.typ.Elem().Kind()](val) + if !convertedVal.IsValid() { + errs.merge(MultiError{"default-" + f.name: fmt.Errorf("failed setting default: %s is not compatible with field %s type", val, f.name)}) + break + } + defaultSlice = reflect.Append(defaultSlice, convertedVal) + } + vCurrent.Set(defaultSlice) + } else if f.typ.Kind() == reflect.Ptr { + t1 := f.typ.Elem() + + if t1.Kind() == reflect.Struct || t1.Kind() == reflect.Slice { + errs.merge(MultiError{"default-" + f.name: errors.New("default option is supported only on: bool, float variants, string, unit variants types or their corresponding pointers or slices")}) + } + + // this check is to handle if the wrong value is provided + if convertedVal := convertPointer(t1.Kind(), f.defaultValue); convertedVal.IsValid() { + vCurrent.Set(convertedVal) + } + } else { + // this check is to handle if the wrong value is provided + if convertedVal := builtinConverters[f.typ.Kind()](f.defaultValue); convertedVal.IsValid() { + vCurrent.Set(builtinConverters[f.typ.Kind()](f.defaultValue)) + } + } + } + } + + return errs +} + +func isPointerToStruct(v reflect.Value) bool { + return !v.IsZero() && v.Type().Kind() == reflect.Ptr && v.Elem().Type().Kind() == reflect.Struct +} + // checkRequired checks whether required fields are empty // // check type t recursively if t has struct fields. @@ -193,7 +276,7 @@ func (d *Decoder) decode(v reflect.Value, path string, parts []pathPart, values if v.Type().Kind() == reflect.Struct { for i := 0; i < v.NumField(); i++ { field := v.Field(i) - if field.Type().Kind() == reflect.Ptr && field.IsNil() && v.Type().Field(i).Anonymous == true { + if field.Type().Kind() == reflect.Ptr && field.IsNil() && v.Type().Field(i).Anonymous { field.Set(reflect.New(field.Type().Elem())) } } diff --git a/vendor/github.com/gorilla/schema/encoder.go b/vendor/github.com/gorilla/schema/encoder.go index f0ed6312..52f2c108 100644 --- a/vendor/github.com/gorilla/schema/encoder.go +++ b/vendor/github.com/gorilla/schema/encoder.go @@ -93,8 +93,11 @@ func (e *Encoder) encode(v reflect.Value, dst map[string][]string) error { } // Encode struct pointer types if the field is a valid pointer and a struct. - if isValidStructPointer(v.Field(i)) { - e.encode(v.Field(i).Elem(), dst) + if isValidStructPointer(v.Field(i)) && !e.hasCustomEncoder(v.Field(i).Type()) { + err := e.encode(v.Field(i).Elem(), dst) + if err != nil { + errors[v.Field(i).Elem().Type().String()] = err + } continue } @@ -112,7 +115,10 @@ func (e *Encoder) encode(v reflect.Value, dst map[string][]string) error { } if v.Field(i).Type().Kind() == reflect.Struct { - e.encode(v.Field(i), dst) + err := e.encode(v.Field(i), dst) + if err != nil { + errors[v.Field(i).Type().String()] = err + } continue } @@ -142,6 +148,11 @@ func (e *Encoder) encode(v reflect.Value, dst map[string][]string) error { return nil } +func (e *Encoder) hasCustomEncoder(t reflect.Type) bool { + _, exists := e.regenc[t] + return exists +} + func typeEncoder(t reflect.Type, reg map[reflect.Type]encoderFunc) encoderFunc { if f, ok := reg[t]; ok { return f diff --git a/vendor/github.com/joho/godotenv/.gitignore b/vendor/github.com/joho/godotenv/.gitignore new file mode 100644 index 00000000..e43b0f98 --- /dev/null +++ b/vendor/github.com/joho/godotenv/.gitignore @@ -0,0 +1 @@ +.DS_Store diff --git a/vendor/github.com/joho/godotenv/LICENCE b/vendor/github.com/joho/godotenv/LICENCE new file mode 100644 index 00000000..e7ddd51b --- /dev/null +++ b/vendor/github.com/joho/godotenv/LICENCE @@ -0,0 +1,23 @@ +Copyright (c) 2013 John Barton + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/vendor/github.com/joho/godotenv/README.md b/vendor/github.com/joho/godotenv/README.md new file mode 100644 index 00000000..bfbe66a0 --- /dev/null +++ b/vendor/github.com/joho/godotenv/README.md @@ -0,0 +1,202 @@ +# GoDotEnv ![CI](https://github.com/joho/godotenv/workflows/CI/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/joho/godotenv)](https://goreportcard.com/report/github.com/joho/godotenv) + +A Go (golang) port of the Ruby [dotenv](https://github.com/bkeepers/dotenv) project (which loads env vars from a .env file). + +From the original Library: + +> Storing configuration in the environment is one of the tenets of a twelve-factor app. Anything that is likely to change between deployment environments–such as resource handles for databases or credentials for external services–should be extracted from the code into environment variables. +> +> But it is not always practical to set environment variables on development machines or continuous integration servers where multiple projects are run. Dotenv load variables from a .env file into ENV when the environment is bootstrapped. + +It can be used as a library (for loading in env for your own daemons etc.) or as a bin command. + +There is test coverage and CI for both linuxish and Windows environments, but I make no guarantees about the bin version working on Windows. + +## Installation + +As a library + +```shell +go get github.com/joho/godotenv +``` + +or if you want to use it as a bin command + +go >= 1.17 +```shell +go install github.com/joho/godotenv/cmd/godotenv@latest +``` + +go < 1.17 +```shell +go get github.com/joho/godotenv/cmd/godotenv +``` + +## Usage + +Add your application configuration to your `.env` file in the root of your project: + +```shell +S3_BUCKET=YOURS3BUCKET +SECRET_KEY=YOURSECRETKEYGOESHERE +``` + +Then in your Go app you can do something like + +```go +package main + +import ( + "log" + "os" + + "github.com/joho/godotenv" +) + +func main() { + err := godotenv.Load() + if err != nil { + log.Fatal("Error loading .env file") + } + + s3Bucket := os.Getenv("S3_BUCKET") + secretKey := os.Getenv("SECRET_KEY") + + // now do something with s3 or whatever +} +``` + +If you're even lazier than that, you can just take advantage of the autoload package which will read in `.env` on import + +```go +import _ "github.com/joho/godotenv/autoload" +``` + +While `.env` in the project root is the default, you don't have to be constrained, both examples below are 100% legit + +```go +godotenv.Load("somerandomfile") +godotenv.Load("filenumberone.env", "filenumbertwo.env") +``` + +If you want to be really fancy with your env file you can do comments and exports (below is a valid env file) + +```shell +# I am a comment and that is OK +SOME_VAR=someval +FOO=BAR # comments at line end are OK too +export BAR=BAZ +``` + +Or finally you can do YAML(ish) style + +```yaml +FOO: bar +BAR: baz +``` + +as a final aside, if you don't want godotenv munging your env you can just get a map back instead + +```go +var myEnv map[string]string +myEnv, err := godotenv.Read() + +s3Bucket := myEnv["S3_BUCKET"] +``` + +... or from an `io.Reader` instead of a local file + +```go +reader := getRemoteFile() +myEnv, err := godotenv.Parse(reader) +``` + +... or from a `string` if you so desire + +```go +content := getRemoteFileContent() +myEnv, err := godotenv.Unmarshal(content) +``` + +### Precedence & Conventions + +Existing envs take precedence of envs that are loaded later. + +The [convention](https://github.com/bkeepers/dotenv#what-other-env-files-can-i-use) +for managing multiple environments (i.e. development, test, production) +is to create an env named `{YOURAPP}_ENV` and load envs in this order: + +```go +env := os.Getenv("FOO_ENV") +if "" == env { + env = "development" +} + +godotenv.Load(".env." + env + ".local") +if "test" != env { + godotenv.Load(".env.local") +} +godotenv.Load(".env." + env) +godotenv.Load() // The Original .env +``` + +If you need to, you can also use `godotenv.Overload()` to defy this convention +and overwrite existing envs instead of only supplanting them. Use with caution. + +### Command Mode + +Assuming you've installed the command as above and you've got `$GOPATH/bin` in your `$PATH` + +``` +godotenv -f /some/path/to/.env some_command with some args +``` + +If you don't specify `-f` it will fall back on the default of loading `.env` in `PWD` + +By default, it won't override existing environment variables; you can do that with the `-o` flag. + +### Writing Env Files + +Godotenv can also write a map representing the environment to a correctly-formatted and escaped file + +```go +env, err := godotenv.Unmarshal("KEY=value") +err := godotenv.Write(env, "./.env") +``` + +... or to a string + +```go +env, err := godotenv.Unmarshal("KEY=value") +content, err := godotenv.Marshal(env) +``` + +## Contributing + +Contributions are welcome, but with some caveats. + +This library has been declared feature complete (see [#182](https://github.com/joho/godotenv/issues/182) for background) and will not be accepting issues or pull requests adding new functionality or breaking the library API. + +Contributions would be gladly accepted that: + +* bring this library's parsing into closer compatibility with the mainline dotenv implementations, in particular [Ruby's dotenv](https://github.com/bkeepers/dotenv) and [Node.js' dotenv](https://github.com/motdotla/dotenv) +* keep the library up to date with the go ecosystem (ie CI bumps, documentation changes, changes in the core libraries) +* bug fixes for use cases that pertain to the library's purpose of easing development of codebases deployed into twelve factor environments + +*code changes without tests and references to peer dotenv implementations will not be accepted* + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Added some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create new Pull Request + +## Releases + +Releases should follow [Semver](http://semver.org/) though the first couple of releases are `v1` and `v1.1`. + +Use [annotated tags for all releases](https://github.com/joho/godotenv/issues/30). Example `git tag -a v1.2.1` + +## Who? + +The original library [dotenv](https://github.com/bkeepers/dotenv) was written by [Brandon Keepers](http://opensoul.org/), and this port was done by [John Barton](https://johnbarton.co/) based off the tests/fixtures in the original library. diff --git a/vendor/github.com/joho/godotenv/godotenv.go b/vendor/github.com/joho/godotenv/godotenv.go new file mode 100644 index 00000000..61b0ebba --- /dev/null +++ b/vendor/github.com/joho/godotenv/godotenv.go @@ -0,0 +1,228 @@ +// Package godotenv is a go port of the ruby dotenv library (https://github.com/bkeepers/dotenv) +// +// Examples/readme can be found on the GitHub page at https://github.com/joho/godotenv +// +// The TL;DR is that you make a .env file that looks something like +// +// SOME_ENV_VAR=somevalue +// +// and then in your go code you can call +// +// godotenv.Load() +// +// and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR") +package godotenv + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "sort" + "strconv" + "strings" +) + +const doubleQuoteSpecialChars = "\\\n\r\"!$`" + +// Parse reads an env file from io.Reader, returning a map of keys and values. +func Parse(r io.Reader) (map[string]string, error) { + var buf bytes.Buffer + _, err := io.Copy(&buf, r) + if err != nil { + return nil, err + } + + return UnmarshalBytes(buf.Bytes()) +} + +// Load will read your env file(s) and load them into ENV for this process. +// +// Call this function as close as possible to the start of your program (ideally in main). +// +// If you call Load without any args it will default to loading .env in the current path. +// +// You can otherwise tell it which files to load (there can be more than one) like: +// +// godotenv.Load("fileone", "filetwo") +// +// It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults. +func Load(filenames ...string) (err error) { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + err = loadFile(filename, false) + if err != nil { + return // return early on a spazout + } + } + return +} + +// Overload will read your env file(s) and load them into ENV for this process. +// +// Call this function as close as possible to the start of your program (ideally in main). +// +// If you call Overload without any args it will default to loading .env in the current path. +// +// You can otherwise tell it which files to load (there can be more than one) like: +// +// godotenv.Overload("fileone", "filetwo") +// +// It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefully set all vars. +func Overload(filenames ...string) (err error) { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + err = loadFile(filename, true) + if err != nil { + return // return early on a spazout + } + } + return +} + +// Read all env (with same file loading semantics as Load) but return values as +// a map rather than automatically writing values into env +func Read(filenames ...string) (envMap map[string]string, err error) { + filenames = filenamesOrDefault(filenames) + envMap = make(map[string]string) + + for _, filename := range filenames { + individualEnvMap, individualErr := readFile(filename) + + if individualErr != nil { + err = individualErr + return // return early on a spazout + } + + for key, value := range individualEnvMap { + envMap[key] = value + } + } + + return +} + +// Unmarshal reads an env file from a string, returning a map of keys and values. +func Unmarshal(str string) (envMap map[string]string, err error) { + return UnmarshalBytes([]byte(str)) +} + +// UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values. +func UnmarshalBytes(src []byte) (map[string]string, error) { + out := make(map[string]string) + err := parseBytes(src, out) + + return out, err +} + +// Exec loads env vars from the specified filenames (empty map falls back to default) +// then executes the cmd specified. +// +// Simply hooks up os.Stdin/err/out to the command and calls Run(). +// +// If you want more fine grained control over your command it's recommended +// that you use `Load()`, `Overload()` or `Read()` and the `os/exec` package yourself. +func Exec(filenames []string, cmd string, cmdArgs []string, overload bool) error { + op := Load + if overload { + op = Overload + } + if err := op(filenames...); err != nil { + return err + } + + command := exec.Command(cmd, cmdArgs...) + command.Stdin = os.Stdin + command.Stdout = os.Stdout + command.Stderr = os.Stderr + return command.Run() +} + +// Write serializes the given environment and writes it to a file. +func Write(envMap map[string]string, filename string) error { + content, err := Marshal(envMap) + if err != nil { + return err + } + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + _, err = file.WriteString(content + "\n") + if err != nil { + return err + } + return file.Sync() +} + +// Marshal outputs the given environment as a dotenv-formatted environment file. +// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped. +func Marshal(envMap map[string]string) (string, error) { + lines := make([]string, 0, len(envMap)) + for k, v := range envMap { + if d, err := strconv.Atoi(v); err == nil { + lines = append(lines, fmt.Sprintf(`%s=%d`, k, d)) + } else { + lines = append(lines, fmt.Sprintf(`%s="%s"`, k, doubleQuoteEscape(v))) + } + } + sort.Strings(lines) + return strings.Join(lines, "\n"), nil +} + +func filenamesOrDefault(filenames []string) []string { + if len(filenames) == 0 { + return []string{".env"} + } + return filenames +} + +func loadFile(filename string, overload bool) error { + envMap, err := readFile(filename) + if err != nil { + return err + } + + currentEnv := map[string]bool{} + rawEnv := os.Environ() + for _, rawEnvLine := range rawEnv { + key := strings.Split(rawEnvLine, "=")[0] + currentEnv[key] = true + } + + for key, value := range envMap { + if !currentEnv[key] || overload { + _ = os.Setenv(key, value) + } + } + + return nil +} + +func readFile(filename string) (envMap map[string]string, err error) { + file, err := os.Open(filename) + if err != nil { + return + } + defer file.Close() + + return Parse(file) +} + +func doubleQuoteEscape(line string) string { + for _, c := range doubleQuoteSpecialChars { + toReplace := "\\" + string(c) + if c == '\n' { + toReplace = `\n` + } + if c == '\r' { + toReplace = `\r` + } + line = strings.Replace(line, string(c), toReplace, -1) + } + return line +} diff --git a/vendor/github.com/joho/godotenv/parser.go b/vendor/github.com/joho/godotenv/parser.go new file mode 100644 index 00000000..cc709af8 --- /dev/null +++ b/vendor/github.com/joho/godotenv/parser.go @@ -0,0 +1,271 @@ +package godotenv + +import ( + "bytes" + "errors" + "fmt" + "regexp" + "strings" + "unicode" +) + +const ( + charComment = '#' + prefixSingleQuote = '\'' + prefixDoubleQuote = '"' + + exportPrefix = "export" +) + +func parseBytes(src []byte, out map[string]string) error { + src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1) + cutset := src + for { + cutset = getStatementStart(cutset) + if cutset == nil { + // reached end of file + break + } + + key, left, err := locateKeyName(cutset) + if err != nil { + return err + } + + value, left, err := extractVarValue(left, out) + if err != nil { + return err + } + + out[key] = value + cutset = left + } + + return nil +} + +// getStatementPosition returns position of statement begin. +// +// It skips any comment line or non-whitespace character. +func getStatementStart(src []byte) []byte { + pos := indexOfNonSpaceChar(src) + if pos == -1 { + return nil + } + + src = src[pos:] + if src[0] != charComment { + return src + } + + // skip comment section + pos = bytes.IndexFunc(src, isCharFunc('\n')) + if pos == -1 { + return nil + } + + return getStatementStart(src[pos:]) +} + +// locateKeyName locates and parses key name and returns rest of slice +func locateKeyName(src []byte) (key string, cutset []byte, err error) { + // trim "export" and space at beginning + src = bytes.TrimLeftFunc(src, isSpace) + if bytes.HasPrefix(src, []byte(exportPrefix)) { + trimmed := bytes.TrimPrefix(src, []byte(exportPrefix)) + if bytes.IndexFunc(trimmed, isSpace) == 0 { + src = bytes.TrimLeftFunc(trimmed, isSpace) + } + } + + // locate key name end and validate it in single loop + offset := 0 +loop: + for i, char := range src { + rchar := rune(char) + if isSpace(rchar) { + continue + } + + switch char { + case '=', ':': + // library also supports yaml-style value declaration + key = string(src[0:i]) + offset = i + 1 + break loop + case '_': + default: + // variable name should match [A-Za-z0-9_.] + if unicode.IsLetter(rchar) || unicode.IsNumber(rchar) || rchar == '.' { + continue + } + + return "", nil, fmt.Errorf( + `unexpected character %q in variable name near %q`, + string(char), string(src)) + } + } + + if len(src) == 0 { + return "", nil, errors.New("zero length string") + } + + // trim whitespace + key = strings.TrimRightFunc(key, unicode.IsSpace) + cutset = bytes.TrimLeftFunc(src[offset:], isSpace) + return key, cutset, nil +} + +// extractVarValue extracts variable value and returns rest of slice +func extractVarValue(src []byte, vars map[string]string) (value string, rest []byte, err error) { + quote, hasPrefix := hasQuotePrefix(src) + if !hasPrefix { + // unquoted value - read until end of line + endOfLine := bytes.IndexFunc(src, isLineEnd) + + // Hit EOF without a trailing newline + if endOfLine == -1 { + endOfLine = len(src) + + if endOfLine == 0 { + return "", nil, nil + } + } + + // Convert line to rune away to do accurate countback of runes + line := []rune(string(src[0:endOfLine])) + + // Assume end of line is end of var + endOfVar := len(line) + if endOfVar == 0 { + return "", src[endOfLine:], nil + } + + // Work backwards to check if the line ends in whitespace then + // a comment (ie asdasd # some comment) + for i := endOfVar - 1; i >= 0; i-- { + if line[i] == charComment && i > 0 { + if isSpace(line[i-1]) { + endOfVar = i + break + } + } + } + + trimmed := strings.TrimFunc(string(line[0:endOfVar]), isSpace) + + return expandVariables(trimmed, vars), src[endOfLine:], nil + } + + // lookup quoted string terminator + for i := 1; i < len(src); i++ { + if char := src[i]; char != quote { + continue + } + + // skip escaped quote symbol (\" or \', depends on quote) + if prevChar := src[i-1]; prevChar == '\\' { + continue + } + + // trim quotes + trimFunc := isCharFunc(rune(quote)) + value = string(bytes.TrimLeftFunc(bytes.TrimRightFunc(src[0:i], trimFunc), trimFunc)) + if quote == prefixDoubleQuote { + // unescape newlines for double quote (this is compat feature) + // and expand environment variables + value = expandVariables(expandEscapes(value), vars) + } + + return value, src[i+1:], nil + } + + // return formatted error if quoted string is not terminated + valEndIndex := bytes.IndexFunc(src, isCharFunc('\n')) + if valEndIndex == -1 { + valEndIndex = len(src) + } + + return "", nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex]) +} + +func expandEscapes(str string) string { + out := escapeRegex.ReplaceAllStringFunc(str, func(match string) string { + c := strings.TrimPrefix(match, `\`) + switch c { + case "n": + return "\n" + case "r": + return "\r" + default: + return match + } + }) + return unescapeCharsRegex.ReplaceAllString(out, "$1") +} + +func indexOfNonSpaceChar(src []byte) int { + return bytes.IndexFunc(src, func(r rune) bool { + return !unicode.IsSpace(r) + }) +} + +// hasQuotePrefix reports whether charset starts with single or double quote and returns quote character +func hasQuotePrefix(src []byte) (prefix byte, isQuored bool) { + if len(src) == 0 { + return 0, false + } + + switch prefix := src[0]; prefix { + case prefixDoubleQuote, prefixSingleQuote: + return prefix, true + default: + return 0, false + } +} + +func isCharFunc(char rune) func(rune) bool { + return func(v rune) bool { + return v == char + } +} + +// isSpace reports whether the rune is a space character but not line break character +// +// this differs from unicode.IsSpace, which also applies line break as space +func isSpace(r rune) bool { + switch r { + case '\t', '\v', '\f', '\r', ' ', 0x85, 0xA0: + return true + } + return false +} + +func isLineEnd(r rune) bool { + if r == '\n' || r == '\r' { + return true + } + return false +} + +var ( + escapeRegex = regexp.MustCompile(`\\.`) + expandVarRegex = regexp.MustCompile(`(\\)?(\$)(\()?\{?([A-Z0-9_]+)?\}?`) + unescapeCharsRegex = regexp.MustCompile(`\\([^$])`) +) + +func expandVariables(v string, m map[string]string) string { + return expandVarRegex.ReplaceAllStringFunc(v, func(s string) string { + submatch := expandVarRegex.FindStringSubmatch(s) + + if submatch == nil { + return s + } + if submatch[1] == "\\" || submatch[2] == "(" { + return submatch[0][1:] + } else if submatch[4] != "" { + return m[submatch[4]] + } + return s + }) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6377ee4c..036da4c7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -142,12 +142,15 @@ github.com/aws/smithy-go/time github.com/aws/smithy-go/transport/http github.com/aws/smithy-go/transport/http/internal/io github.com/aws/smithy-go/waiter -# github.com/gorilla/schema v1.2.0 -## explicit +# github.com/gorilla/schema v1.4.0 +## explicit; go 1.20 github.com/gorilla/schema # github.com/jmespath/go-jmespath v0.4.0 ## explicit; go 1.14 github.com/jmespath/go-jmespath +# github.com/joho/godotenv v1.5.1 +## explicit; go 1.12 +github.com/joho/godotenv # github.com/lib/pq v1.10.3 ## explicit; go 1.13 github.com/lib/pq