From 1fb6d9a18baf078464db3901345d5b2ecaabaf9d Mon Sep 17 00:00:00 2001 From: Wenbo Han Date: Mon, 12 Aug 2024 15:12:48 +0800 Subject: [PATCH] feat: optimize validation (#594) * feat: optimize validation * update description * remove debug --- console/cli_context.go | 2 +- support/maps/maps.go | 52 +++++++ support/maps/maps_test.go | 38 +++++ validation/validator.go | 50 ++++--- validation/validator_test.go | 265 ++++++++++++++++++++++++----------- 5 files changed, 310 insertions(+), 97 deletions(-) diff --git a/console/cli_context.go b/console/cli_context.go index fb2da0e46..f8ba153b6 100644 --- a/console/cli_context.go +++ b/console/cli_context.go @@ -230,7 +230,7 @@ func (r *CliContext) Secret(question string, option ...console.SecretOption) (st } func (r *CliContext) Spinner(message string, option console.SpinnerOption) error { - style := lipgloss.NewStyle().Foreground(lipgloss.Color("#8ED3F9")) + style := lipgloss.NewStyle().Foreground(lipgloss.CompleteColor{TrueColor: "#3D8C8D", ANSI256: "30", ANSI: "6"}) spin := spinner.New().Title(message).Style(style).TitleStyle(style) var err error diff --git a/support/maps/maps.go b/support/maps/maps.go index 9c59831de..3ea550b52 100644 --- a/support/maps/maps.go +++ b/support/maps/maps.go @@ -1,5 +1,9 @@ package maps +import ( + "reflect" +) + // Add an element to a map if it doesn't exist. func Add[K comparable, V any](mp map[K]V, k K, v V) { if Exists(mp, k) { @@ -25,6 +29,54 @@ func Forget[K comparable, V any](mp map[K]V, keys ...K) { } } +func FromStruct(data any) map[string]any { + res := make(map[string]any) + dataType := reflect.TypeOf(data) + dataValue := reflect.ValueOf(data) + + if dataType.Kind() == reflect.Pointer { + dataType = dataType.Elem() + dataValue = dataValue.Elem() + } + + if dataType.Kind() != reflect.Struct { + return res + } + + for i := 0; i < dataType.NumField(); i++ { + fieldType := dataType.Field(i) + fieldValue := dataValue.Field(i) + + if !fieldType.IsExported() { + continue + } + + if fieldValue.Kind() == reflect.Pointer { + if fieldValue.IsNil() { + res[fieldType.Name] = nil + continue + } + + fieldValue = fieldValue.Elem() + } + + if fieldValue.Kind() == reflect.Struct { + subStructMap := FromStruct(fieldValue.Interface()) + if fieldType.Anonymous { + for key, value := range subStructMap { + res[key] = value + } + } else { + res[fieldType.Name] = subStructMap + } + } else { + res[fieldType.Name] = fieldValue.Interface() + } + } + + return res +} + // Get an element from a map func Get[K comparable, V any](mp map[K]V, key K, defaults ...V) V { val, ok := mp[key] diff --git a/support/maps/maps_test.go b/support/maps/maps_test.go index 6f9430391..591de967a 100644 --- a/support/maps/maps_test.go +++ b/support/maps/maps_test.go @@ -91,6 +91,44 @@ func TestForget(t *testing.T) { }, gMp) } +func TestFromStruct(t *testing.T) { + type One struct { + Name string + Age int + } + type Two struct { + Height int + } + type Three struct { + Two + One One + Name string + age int + } + data := Three{ + Name: "Three", + Two: Two{ + Height: 1, + }, + One: One{ + Name: "One", + Age: 18, + }, + age: 1, + } + + res := FromStruct(data) + + assert.Equal(t, "Three", res["Name"]) + assert.Equal(t, 1, res["Height"]) + + one, ok := res["One"].(map[string]any) + + assert.True(t, ok) + assert.Equal(t, "One", one["Name"]) + assert.Equal(t, 18, one["Age"]) +} + func TestGet(t *testing.T) { mp := map[string]any{ "name": "Krishan", diff --git a/validation/validator.go b/validation/validator.go index 726010f1d..818818180 100644 --- a/validation/validator.go +++ b/validation/validator.go @@ -1,8 +1,8 @@ package validation import ( + "net/url" "reflect" - "strings" "github.com/gookit/validate" "github.com/mitchellh/mapstructure" @@ -10,7 +10,7 @@ import ( httpvalidate "github.com/goravel/framework/contracts/validation" "github.com/goravel/framework/support/carbon" - "github.com/goravel/framework/support/str" + "github.com/goravel/framework/support/maps" ) func init() { @@ -38,21 +38,39 @@ func (v *Validator) Bind(ptr any) error { return nil } + // SafeData only contains the data that is defined in the rules, + // we want user can the original data that is not defined in the rules, + // so that user doesn't need to define rules for all fields. data := v.instance.SafeData() - // When rules check slice: "field.*", SafeData will only have the "field.*" key, doesn't have the field key. - for key, value := range data { - if str.Of(key).EndsWith(".*") { - realKey := strings.ReplaceAll(key, ".*", "") - if _, exist := data[realKey]; !exist { - data[realKey] = value + if formData, ok := v.data.(*validate.FormData); ok { + if values, ok := v.data.Src().(url.Values); ok { + for key, value := range values { + if _, exist := data[key]; !exist { + data[key] = value[0] + } } - } - } - if formData, ok := v.data.(*validate.FormData); ok { - for key, value := range formData.Files { - data[key] = value + for key, value := range formData.Files { + if _, exist := data[key]; !exist { + data[key] = value + } + } + } + } else if _, ok := v.data.(*validate.MapData); ok { + values := v.data.Src().(map[string]any) + for key, value := range values { + if _, exist := data[key]; !exist { + data[key] = value + } + } + } else { + if srcMap := maps.FromStruct(v.data.Src()); len(srcMap) > 0 { + for key, value := range srcMap { + if _, exist := data[key]; !exist { + data[key] = value + } + } } } @@ -65,11 +83,7 @@ func (v *Validator) Bind(ptr any) error { return err } - if err = decoder.Decode(data); err != nil { - return err - } - - return nil + return decoder.Decode(data) } func (v *Validator) Errors() httpvalidate.Errors { diff --git a/validation/validator_test.go b/validation/validator_test.go index d9a59b7ff..2c9a54633 100644 --- a/validation/validator_test.go +++ b/validation/validator_test.go @@ -19,7 +19,7 @@ import ( "github.com/goravel/framework/support/carbon" ) -func TestBind(t *testing.T) { +func TestBind_Rule(t *testing.T) { type Data struct { A string `form:"a" json:"a"` B int `form:"b" json:"b"` @@ -42,24 +42,22 @@ func TestBind(t *testing.T) { } tests := []struct { - name string - data validate.DataFace - rules map[string]string - filters map[string]string - assert func(data Data) + name string + data validate.DataFace + rules map[string]string + assert func(data Data) }{ { - name: "success when data is map and key is lowercase", - data: validate.FromMap(map[string]any{"a": "aa ", "b": "1"}), - rules: map[string]string{"a": "required", "b": "required"}, - filters: map[string]string{"a": "trim", "b": "int"}, + name: "data is map and key is lowercase", + data: validate.FromMap(map[string]any{"a": "aa", "b": "1"}), + rules: map[string]string{"a": "required"}, assert: func(data Data) { assert.Equal(t, "aa", data.A) assert.Equal(t, 1, data.B) }, }, { - name: "success when data is map and cast key", + name: "data is map and cast key", data: validate.FromMap(map[string]any{"b": "1"}), rules: map[string]string{"b": "required"}, assert: func(data Data) { @@ -67,53 +65,53 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is map and key is uppercase", - data: validate.FromMap(map[string]any{"A": "aa "}), - rules: map[string]string{"A": "required"}, - filters: map[string]string{"A": "trim"}, + name: "data is map and key is uppercase", + data: validate.FromMap(map[string]any{"A": "aa"}), + rules: map[string]string{"A": "required"}, assert: func(data Data) { assert.Equal(t, "aa", data.A) }, }, { - name: "success when data is struct", + name: "data is struct", data: func() validate.DataFace { data, err := validate.FromStruct(&struct { A string + B int }{ - A: "aa", + A: "a", + B: 1, }) assert.Nil(t, err) return data }(), - rules: map[string]string{"A": "required"}, - filters: map[string]string{"A": "trim"}, + rules: map[string]string{"A": "required"}, assert: func(data Data) { - assert.Equal(t, "aa", data.A) + assert.Equal(t, "a", data.A) + assert.Equal(t, 1, data.B) }, }, { - name: "success when data is get request", + name: "data is get request", data: func() validate.DataFace { - request, err := http.NewRequest(http.MethodGet, "/?a=aa &&b=1", nil) + request, err := http.NewRequest(http.MethodGet, "/?a=aa&&b=1", nil) assert.Nil(t, err) data, err := validate.FromRequest(request) assert.Nil(t, err) return data }(), - rules: map[string]string{"a": "required", "b": "required"}, - filters: map[string]string{"a": "trim"}, + rules: map[string]string{"a": "required"}, assert: func(data Data) { assert.Equal(t, "aa", data.A) assert.Equal(t, 1, data.B) }, }, { - name: "success when data is post request", + name: "data is post request", data: func() validate.DataFace { - request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"a":"Goravel", "ages": [1, 2], "names": ["a", "b"]}`)) + request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"a":"Goravel", "b": 1, "ages": [1, 2], "names": ["a", "b"]}`)) request.Header.Set("Content-Type", "application/json") assert.Nil(t, err) data, err := validate.FromRequest(request) @@ -130,12 +128,13 @@ func TestBind(t *testing.T) { rules: map[string]string{"a": "required", "ages.*": "int", "names.*": "string"}, assert: func(data Data) { assert.Equal(t, "Goravel", data.A) + assert.Equal(t, 1, data.B) assert.Equal(t, []int{1, 2}, data.Ages) assert.Equal(t, []string{"a", "b"}, data.Names) }, }, { - name: "success when data is post request with Carbon", + name: "data is post request with Carbon", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"carbon": "2024-07-04 10:00:52"}`)) request.Header.Set("Content-Type", "application/json") @@ -152,7 +151,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTime", + name: "data is post request with DateTime", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time": "2024-07-04 10:00:52"}`)) request.Header.Set("Content-Type", "application/json") @@ -169,7 +168,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTime(string)", + name: "data is post request with DateTime(string)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time": "1720087252"}`)) request.Header.Set("Content-Type", "application/json") @@ -186,7 +185,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTime(int)", + name: "data is post request with DateTime(int)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time": 1720087252}`)) request.Header.Set("Content-Type", "application/json") @@ -203,7 +202,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTime(milli)", + name: "data is post request with DateTime(milli)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time": 1720087252000}`)) request.Header.Set("Content-Type", "application/json") @@ -220,7 +219,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTime(micro)", + name: "data is post request with DateTime(micro)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time": 1720087252000000}`)) request.Header.Set("Content-Type", "application/json") @@ -237,7 +236,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTime(nano)", + name: "data is post request with DateTime(nano)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time": 1720087252000000000}`)) request.Header.Set("Content-Type", "application/json") @@ -254,7 +253,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTimeMilli", + name: "data is post request with DateTimeMilli", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time_milli": "2024-07-04 10:00:52.123"}`)) request.Header.Set("Content-Type", "application/json") @@ -271,7 +270,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTimeMilli(int)", + name: "data is post request with DateTimeMilli(int)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time_milli": 1720087252123}`)) request.Header.Set("Content-Type", "application/json") @@ -288,7 +287,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTimeMicro", + name: "data is post request with DateTimeMicro", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time_micro": "2024-07-04 10:00:52.123456"}`)) request.Header.Set("Content-Type", "application/json") @@ -305,7 +304,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTimeNano", + name: "data is post request with DateTimeNano", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time_nano": "2024-07-04 10:00:52.123456789"}`)) request.Header.Set("Content-Type", "application/json") @@ -322,7 +321,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateTimeNano(int)", + name: "data is post request with DateTimeNano(int)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_time_nano": "1720087252123456789"}`)) request.Header.Set("Content-Type", "application/json") @@ -339,7 +338,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with Date", + name: "data is post request with Date", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date": "2024-07-04"}`)) request.Header.Set("Content-Type", "application/json") @@ -356,7 +355,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with Date(int)", + name: "data is post request with Date(int)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date": 1720087252}`)) request.Header.Set("Content-Type", "application/json") @@ -373,7 +372,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateMilli", + name: "data is post request with DateMilli", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_milli": "2024-07-04.123"}`)) request.Header.Set("Content-Type", "application/json") @@ -390,7 +389,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateMilli(int)", + name: "data is post request with DateMilli(int)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_milli": 1720087252123}`)) request.Header.Set("Content-Type", "application/json") @@ -407,7 +406,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateMicro", + name: "data is post request with DateMicro", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_micro": "2024-07-04.123456"}`)) request.Header.Set("Content-Type", "application/json") @@ -424,7 +423,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateMicro(int)", + name: "data is post request with DateMicro(int)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_micro": 1720087252123456}`)) request.Header.Set("Content-Type", "application/json") @@ -441,7 +440,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateNano", + name: "data is post request with DateNano", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_nano": "2024-07-04.123456789"}`)) request.Header.Set("Content-Type", "application/json") @@ -458,7 +457,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with DateNano(int)", + name: "data is post request with DateNano(int)", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"date_nano": "1720087252123456789"}`)) request.Header.Set("Content-Type", "application/json") @@ -475,7 +474,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with Timestamp", + name: "data is post request with Timestamp", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"timestamp": 1720087252}`)) request.Header.Set("Content-Type", "application/json") @@ -492,7 +491,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with TimestampMilli", + name: "data is post request with TimestampMilli", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"timestamp_milli": 1720087252123}`)) request.Header.Set("Content-Type", "application/json") @@ -509,7 +508,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with TimestampMicro", + name: "data is post request with TimestampMicro", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"timestamp_micro": 1720087252123456}`)) request.Header.Set("Content-Type", "application/json") @@ -526,7 +525,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with TimestampNano", + name: "data is post request with TimestampNano", data: func() validate.DataFace { request, err := http.NewRequest(http.MethodPost, "/", bytes.NewBufferString(`{"timestamp_nano": "1720087252123456789"}`)) request.Header.Set("Content-Type", "application/json") @@ -543,7 +542,7 @@ func TestBind(t *testing.T) { }, }, { - name: "success when data is post request with body", + name: "data is post request with body", data: func() validate.DataFace { request := buildRequest(t) data, err := validate.FromRequest(request, 1) @@ -551,8 +550,7 @@ func TestBind(t *testing.T) { return data }(), - rules: map[string]string{"a": "required", "file": "file"}, - filters: map[string]string{"a": "trim"}, + rules: map[string]string{"a": "required", "file": "file"}, assert: func(data Data) { request := buildRequest(t) _, file, err := request.FormFile("file") @@ -565,6 +563,117 @@ func TestBind(t *testing.T) { }, } + validation := NewValidation() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + validator, err := validation.Make(test.data, test.rules) + require.Nil(t, err) + require.Nil(t, validator.Errors()) + + var data Data + err = validator.Bind(&data) + require.Nil(t, err) + + test.assert(data) + }) + } +} + +func TestBind_Filter(t *testing.T) { + type Data struct { + A string `form:"a" json:"a"` + B int `form:"b" json:"b"` + } + + tests := []struct { + name string + data validate.DataFace + rules map[string]string + filters map[string]string + assert func(data Data) + }{ + { + name: "data is map and key is lowercase", + data: validate.FromMap(map[string]any{"a": " a ", "b": "1"}), + rules: map[string]string{"a": "required", "b": "required"}, + filters: map[string]string{"a": "trim", "b": "int"}, + assert: func(data Data) { + assert.Equal(t, "a", data.A) + assert.Equal(t, 1, data.B) + }, + }, + { + name: "data is map and key is lowercase, a no rule but has filter, a should keep the original value.", + data: validate.FromMap(map[string]any{"a": "a", "b": " 1"}), + rules: map[string]string{"b": "required"}, + filters: map[string]string{"a": "upper", "b": "trim|int"}, + assert: func(data Data) { + assert.Equal(t, "a", data.A) + assert.Equal(t, 1, data.B) + }, + }, + { + name: "data is struct", + data: func() validate.DataFace { + data, err := validate.FromStruct(&struct { + A string + }{ + A: " a ", + }) + assert.Nil(t, err) + + return data + }(), + rules: map[string]string{"A": "required"}, + filters: map[string]string{"A": "trim"}, + assert: func(data Data) { + assert.Equal(t, "a", data.A) + }, + }, + { + name: "data is get request", + data: func() validate.DataFace { + request, err := http.NewRequest(http.MethodGet, "/?a= a &&b=1", nil) + assert.Nil(t, err) + data, err := validate.FromRequest(request) + assert.Nil(t, err) + + return data + }(), + rules: map[string]string{"a": "required", "b": "required"}, + filters: map[string]string{"a": "trim"}, + assert: func(data Data) { + assert.Equal(t, "a", data.A) + assert.Equal(t, 1, data.B) + }, + }, + { + name: "data is post request with body", + data: func() validate.DataFace { + payload := &bytes.Buffer{} + writer := multipart.NewWriter(payload) + + err := writer.WriteField("a", " a ") + assert.Nil(t, err) + assert.Nil(t, writer.Close()) + + request, err := http.NewRequest(http.MethodPost, "/", payload) + assert.Nil(t, err) + request.Header.Set("Content-Type", writer.FormDataContentType()) + + data, err := validate.FromRequest(request, 1) + assert.Nil(t, err) + + return data + }(), + rules: map[string]string{"a": "required", "file": "file"}, + filters: map[string]string{"a": "trim"}, + assert: func(data Data) { + assert.Equal(t, "a", data.A) + }, + }, + } + validation := NewValidation() for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -813,31 +922,6 @@ func TestCastValue(t *testing.T) { } } -func buildRequest(t *testing.T) *http.Request { - payload := &bytes.Buffer{} - writer := multipart.NewWriter(payload) - - err := writer.WriteField("a", "aa") - assert.Nil(t, err) - - logo, err := os.Open("../logo.png") - assert.Nil(t, err) - - defer logo.Close() - part1, err := writer.CreateFormFile("file", filepath.Base("../logo.png")) - assert.Nil(t, err) - - _, err = io.Copy(part1, logo) - assert.Nil(t, err) - assert.Nil(t, writer.Close()) - - request, err := http.NewRequest(http.MethodPost, "/", payload) - assert.Nil(t, err) - request.Header.Set("Content-Type", writer.FormDataContentType()) - - return request -} - func TestCastCarbon(t *testing.T) { tests := []struct { name string @@ -984,3 +1068,28 @@ func TestCastCarbon(t *testing.T) { }) } } + +func buildRequest(t *testing.T) *http.Request { + payload := &bytes.Buffer{} + writer := multipart.NewWriter(payload) + + err := writer.WriteField("a", "aa") + assert.Nil(t, err) + + logo, err := os.Open("../logo.png") + assert.Nil(t, err) + + defer logo.Close() + part1, err := writer.CreateFormFile("file", filepath.Base("../logo.png")) + assert.Nil(t, err) + + _, err = io.Copy(part1, logo) + assert.Nil(t, err) + assert.Nil(t, writer.Close()) + + request, err := http.NewRequest(http.MethodPost, "/", payload) + assert.Nil(t, err) + request.Header.Set("Content-Type", writer.FormDataContentType()) + + return request +}