diff --git a/CHANGELOG.md b/CHANGELOG.md index e1eee381..f861b117 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,11 @@ Types of changes - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [1.29.0] + +- `Added` mask `apply` to externalize masks +- `Fixed` mock command fails to parse JSON body when root is an array + ## [1.28.1] - `Fixed` mock command fails when log level is not TRACE diff --git a/internal/app/pimo/mock.go b/internal/app/pimo/mock.go deleted file mode 100644 index bd620dcb..00000000 --- a/internal/app/pimo/mock.go +++ /dev/null @@ -1,428 +0,0 @@ -package pimo - -import ( - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "strings" - - "github.com/cgi-fr/pimo/pkg/jsonline" - "github.com/cgi-fr/pimo/pkg/model" -) - -type RequestDict struct { - model.Dictionary -} - -type ResponseDict struct { - model.Dictionary -} - -const ( - keyMethod = "method" - keyStatus = "status" - keyURL = "url" - // following properties are always empty - // keyURLScheme = "scheme" - // keyURLUser = "user" - // keyURLUserName = "name" - // keyURLUserPassword = "pass" - // keyURLHost = "host" - // keyURLHostName = "name" - // keyURLHostPort = "port" - keyURLPath = "path" - keyURLQuery = "query" - keyURLFragment = "fragment" - keyProtocol = "protocol" - keyHeaders = "headers" - keyBody = "body" - keyTrailers = "trailers" -) - -func (r RequestDict) Method() string { - return r.UnpackAsDict().Get(keyMethod).(string) -} - -// func (r RequestDict) URLScheme() string { -// return r.UnpackAsDict().Get(keyURL).(model.Dictionary).Get(keyURLScheme).(string) -// } - -// func (r RequestDict) URLUser() (string, bool) { -// s, ok := r.UnpackAsDict().Get(keyURL).(model.Dictionary).Get(keyURLUser).(model.Dictionary).Get(keyURLUserName).(string) -// return s, ok -// } - -// func (r RequestDict) URLPassword() (string, bool) { -// s, ok := r.UnpackAsDict().Get(keyURL).(model.Dictionary).Get(keyURLUser).(model.Dictionary).Get(keyURLUserPassword).(string) -// return s, ok -// } - -// func (r RequestDict) URLHostName() string { -// return r.UnpackAsDict().Get(keyURL).(model.Dictionary).Get(keyURLHost).(model.Dictionary).Get(keyURLHostName).(string) -// } - -// func (r RequestDict) URLHostPort() (string, bool) { -// s, ok := r.UnpackAsDict().Get(keyURL).(model.Dictionary).Get(keyURLHost).(model.Dictionary).Get(keyURLHostPort).(string) -// return s, ok -// } - -func (r RequestDict) URLPath() string { - return r.UnpackAsDict().Get(keyURL).(model.Dictionary).Get(keyURLPath).(string) -} - -func (r RequestDict) URLFragment() (string, bool) { - s, ok := r.UnpackAsDict().Get(keyURL).(model.Dictionary).Get(keyURLFragment).(string) - return s, ok -} - -func (r RequestDict) Protocol() string { - return r.UnpackAsDict().Get(keyProtocol).(string) -} - -func (r RequestDict) Body() string { - return r.UnpackAsDict().Get(keyBody).(string) -} - -func NewRequestDict(request *http.Request) (RequestDict, error) { - dict := model.NewDictionary() - dict.Set(keyMethod, request.Method) - dict.Set(keyURL, urlToDict(request.URL)) - dict.Set(keyProtocol, request.Proto) - headers := model.NewDictionary() - for key, values := range request.Header { - entries := make([]model.Entry, len(values)) - for i, val := range values { - entries[i] = val - } - headers.Set(key, entries) - } - dict.Set(keyHeaders, headers) - - if request.Body != nil { - b, err := io.ReadAll(request.Body) - if err != nil { - return RequestDict{dict.Pack()}, err - } - if bodydict, err := jsonline.JSONToDictionary(b); err != nil { - dict.Set(keyBody, string(b)) - } else { - dict.Set(keyBody, bodydict) - } - } - - return RequestDict{dict.Pack()}, nil -} - -func NewResponseDict(response *http.Response) (ResponseDict, error) { - dict := model.NewDictionary() - - dict.Set(keyStatus, response.StatusCode) - dict.Set(keyProtocol, response.Proto) - - headers := model.NewDictionary() - for key, values := range response.Header { - entries := make([]model.Entry, len(values)) - for i, val := range values { - entries[i] = val - } - headers.Set(key, entries) - } - dict.Set(keyHeaders, headers) - - if response.Body != nil { - b, err := io.ReadAll(response.Body) - if err != nil { - return ResponseDict{dict.Pack()}, err - } - if bodydict, err := jsonline.JSONToDictionary(b); err != nil { - dict.Set(keyBody, string(b)) - } else { - dict.Set(keyBody, bodydict) - } - } - - trailers := model.NewDictionary() - for key, values := range response.Trailer { - entries := make([]model.Entry, len(values)) - for i, val := range values { - entries[i] = val - } - trailers.Set(key, entries) - } - dict.Set(keyTrailers, trailers) - - return ResponseDict{dict.Pack()}, nil -} - -func urlToDict(url *url.URL) model.Dictionary { - dict := model.NewDictionary() - if url == nil { - return dict - } - - // dict.Set(keyURLScheme, url.Scheme) - - // if url.User != nil { - // user := model.NewDictionary().With(keyURLUserName, url.User.Username()) - // if password, set := url.User.Password(); set { - // user.Set(keyURLUserPassword, password) - // } - // dict.Set(keyURLUser, user) - // } - - // host := model.NewDictionary().With(keyURLHostName, url.Hostname()) - // if url.Port() != "" { - // host.Set(keyURLHostPort, url.Port()) - // } - // dict.Set(keyURLHost, host) - - dict.Set(keyURLPath, url.EscapedPath()) - if url.RawQuery != "" { - dict.Set(keyURLQuery, queryToDict(url.RawQuery)) - } - - if url.RawFragment != "" { - dict.Set(keyURLFragment, url.EscapedFragment()) - } - - return dict -} - -func queryToDict(rawquery string) model.Dictionary { - query := model.NewDictionary() - for _, pair := range strings.Split(rawquery, "&") { - if key, value, found := strings.Cut(pair, "="); found { - if values, exists := query.GetValue(key); exists { - query.Set(key, append(values.([]model.Entry), value)) - } else { - query.Set(key, []model.Entry{value}) - } - } else { - if values, exists := query.GetValue(key); exists { - query.Set(key, append(values.([]model.Entry), nil)) - } else { - query.Set(key, []model.Entry{nil}) - } - } - } - return query -} - -func ToRequest(dict model.Dictionary) (*http.Request, error) { - if d, ok := dict.TryUnpackAsDict(); ok { - dict = d - } - - var method string - if m, ok := dict.GetValue(keyMethod); ok { - if s, ok := m.(string); ok { - method = s - } - } - - var url string - if u, ok := dict.Get(keyURL).(model.Dictionary); ok { - var err error - url, err = dictToURL(u) - if err != nil { - return nil, err - } - } - - var body string - if b, ok := dict.GetValue(keyBody); ok { - if s, ok := b.(string); ok { - body = s - } else if d, ok := b.(model.Dictionary); ok { - bytes, err := d.MarshalJSON() - if err != nil { - return nil, err - } - body = string(bytes) - } - } - - r, err := http.NewRequest(method, url, strings.NewReader(body)) - - var headers model.Dictionary - if h, ok := dict.GetValue(keyHeaders); ok { - if d, ok := h.(model.Dictionary); ok { - headers = d - } - } - - for _, key := range headers.Keys() { - if values, ok := headers.Get(key).([]model.Entry); ok { - for _, value := range values { - if s, ok := value.(string); ok { - r.Header.Add(key, s) - } - } - } - } - - if p, ok := dict.GetValue(keyProtocol); ok { - if s, ok := p.(string); ok { - r.Proto = s - } - } - - return r, err -} - -func dictToURL(dict model.Dictionary) (string, error) { - // var user *url.Userinfo - // if userinfo, ok := dict.Get(keyURLUser).(model.Dictionary); ok { - // if pass, ok := userinfo.Get(keyURLUserPassword).(string); ok { - // user = url.UserPassword(userinfo.Get(keyURLUserName).(string), pass) - // } - // } - - // hostport := dict.Get(keyURLHost).(model.Dictionary).Get(keyURLHostName).(string) - // if port, ok := dict.Get(keyURLHost).(model.Dictionary).Get(keyURLHostPort).(string); ok { - // hostport = hostport + ":" + port - // } - - rawpath := dict.Get(keyURLPath).(string) - path, err := url.PathUnescape(rawpath) - if err != nil { - return "", err - } - - queryDict, ok := dict.GetValue(keyURLQuery) - queryRaw := "" - if ok { - queryRaw = dictToRawQuery(queryDict.(model.Dictionary)) - } - - fragmentDict, ok := dict.GetValue(keyURLFragment) - fragmentRaw := "" - fragment := "" - if ok { - fragmentRaw = fragmentDict.(string) - fragment, err = url.PathUnescape(fragmentRaw) - if err != nil { - return "", err - } - } - - url := &url.URL{ - Scheme: "", // dict.Get(keyURLScheme).(string), - Opaque: "", - User: nil, // user, - Host: "", // hostport, - Path: path, - RawPath: rawpath, - OmitHost: false, - ForceQuery: false, - RawQuery: queryRaw, - Fragment: fragment, - RawFragment: fragmentRaw, - } - - return url.String(), nil -} - -func dictToRawQuery(dict model.Dictionary) string { - query := url.Values{} - - for _, key := range dict.Keys() { - values := dict.Get(key).([]model.Entry) - for _, value := range values { - query.Add(key, value.(string)) - } - } - - return query.Encode() -} - -func ToResponse(dict model.Dictionary) (*http.Response, error) { - if d, ok := dict.TryUnpackAsDict(); ok { - dict = d - } - - var status int - if m, ok := dict.GetValue(keyStatus); ok { - if s, ok := m.(int); ok { - status = s - } - } - - var protocol string - if p, ok := dict.GetValue(keyProtocol); ok { - if s, ok := p.(string); ok { - protocol = s - } - } - - var body string - if b, ok := dict.GetValue(keyBody); ok { - if s, ok := b.(string); ok { - body = s - } else if d, ok := b.(model.Dictionary); ok { - bytes, err := d.MarshalJSON() - if err != nil { - return nil, err - } - body = string(bytes) + "\n" - } - } - - response := &http.Response{ - Status: fmt.Sprintf("%d %s", status, http.StatusText(status)), - StatusCode: status, - Proto: protocol, - ProtoMajor: 0, - ProtoMinor: 0, - Header: http.Header{}, - Body: io.NopCloser(strings.NewReader(body)), - ContentLength: 0, - TransferEncoding: nil, - Close: false, - Uncompressed: false, - Trailer: http.Header{}, - Request: nil, - TLS: nil, - } - - var headers model.Dictionary - if h, ok := dict.GetValue(keyHeaders); ok { - if d, ok := h.(model.Dictionary); ok { - headers = d - } - } - - for _, key := range headers.Keys() { - if values, ok := headers.Get(key).([]model.Entry); ok { - for _, value := range values { - if s, ok := value.(string); ok { - if key == "Content-Length" { - s = strconv.Itoa(len(body)) - } - response.Header.Add(key, s) - } - } - } - } - - var trailers model.Dictionary - if h, ok := dict.GetValue(keyTrailers); ok { - if d, ok := h.(model.Dictionary); ok { - trailers = d - } - } - - for _, key := range trailers.Keys() { - if values, ok := trailers.Get(key).([]model.Entry); ok { - for _, value := range values { - if s, ok := value.(string); ok { - response.Trailer.Add(key, s) - } - } - } - } - - return response, nil -} diff --git a/internal/app/pimo/mock/request.go b/internal/app/pimo/mock/request.go index 44b17459..951b1fb3 100644 --- a/internal/app/pimo/mock/request.go +++ b/internal/app/pimo/mock/request.go @@ -6,8 +6,9 @@ import ( "net/url" "strings" - "github.com/cgi-fr/pimo/pkg/jsonline" "github.com/cgi-fr/pimo/pkg/model" + "github.com/goccy/go-json" + "github.com/rs/zerolog/log" ) type RequestDict struct { @@ -40,12 +41,10 @@ func (r RequestDict) ToRequest() (*http.Request, error) { if b, ok := dict.GetValue(keyBody); ok { if s, ok := b.(string); ok { body = s - } else if d, ok := b.(model.Dictionary); ok { - bytes, err := d.MarshalJSON() - if err != nil { - return nil, err - } - body = string(bytes) + } else if bytes, err := json.Marshal(b); err == nil { + body = string(bytes) + "\n" + } else { + log.Err(err).Msg("Failed to read body") } } @@ -98,10 +97,12 @@ func NewRequestDict(request *http.Request) (RequestDict, error) { if err != nil { return RequestDict{dict.Pack()}, err } - if bodydict, err := jsonline.JSONToDictionary(b); err != nil { + + var bodydict interface{} + if err := json.Unmarshal(b, &bodydict); err != nil { dict.Set(keyBody, string(b)) } else { - dict.Set(keyBody, bodydict) + dict.Set(keyBody, model.CleanTypes(bodydict)) } } diff --git a/internal/app/pimo/mock/response.go b/internal/app/pimo/mock/response.go index 516e5bba..53022c02 100644 --- a/internal/app/pimo/mock/response.go +++ b/internal/app/pimo/mock/response.go @@ -7,8 +7,9 @@ import ( "strconv" "strings" - "github.com/cgi-fr/pimo/pkg/jsonline" "github.com/cgi-fr/pimo/pkg/model" + "github.com/goccy/go-json" + "github.com/rs/zerolog/log" ) type ResponseDict struct { @@ -42,12 +43,10 @@ func (r ResponseDict) ToResponse() (*http.Response, error) { if b, ok := dict.GetValue(keyBody); ok { if s, ok := b.(string); ok { body = s - } else if d, ok := b.(model.Dictionary); ok { - bytes, err := d.MarshalJSON() - if err != nil { - return nil, err - } + } else if bytes, err := json.Marshal(b); err == nil { body = string(bytes) + "\n" + } else { + log.Err(err).Msg("Failed to read body") } } @@ -129,10 +128,12 @@ func NewResponseDict(response *http.Response) (ResponseDict, error) { if err != nil { return ResponseDict{dict.Pack()}, err } - if bodydict, err := jsonline.JSONToDictionary(b); err != nil { + + var bodydict interface{} + if err := json.Unmarshal(b, &bodydict); err != nil { dict.Set(keyBody, string(b)) } else { - dict.Set(keyBody, bodydict) + dict.Set(keyBody, model.CleanTypes(bodydict)) } } diff --git a/internal/app/pimo/pimo.go b/internal/app/pimo/pimo.go index aed3beee..2232938d 100755 --- a/internal/app/pimo/pimo.go +++ b/internal/app/pimo/pimo.go @@ -28,6 +28,7 @@ import ( over "github.com/adrienaury/zeromdc" "github.com/cgi-fr/pimo/pkg/add" "github.com/cgi-fr/pimo/pkg/addtransient" + "github.com/cgi-fr/pimo/pkg/apply" "github.com/cgi-fr/pimo/pkg/command" "github.com/cgi-fr/pimo/pkg/constant" "github.com/cgi-fr/pimo/pkg/dateparser" @@ -341,6 +342,7 @@ func injectMaskFactories() []model.MaskFactory { timeline.Factory, sequence.Factory, sha3.Factory, + apply.Factory, } } diff --git a/pkg/apply/apply.go b/pkg/apply/apply.go new file mode 100644 index 00000000..e8f0ef85 --- /dev/null +++ b/pkg/apply/apply.go @@ -0,0 +1,84 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of PIMO. +// +// PIMO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// PIMO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with PIMO. If not, see . + +package apply + +import ( + "hash/fnv" + "text/template" + + "github.com/cgi-fr/pimo/pkg/model" + "github.com/rs/zerolog/log" +) + +// MaskEngine is a value that always mask the same way +type MaskEngine struct { + pipeline model.Pipeline +} + +// NewMask return a MaskEngine from a value +func NewMask(seed int64, caches map[string]model.Cache, fns template.FuncMap, uri string) (MaskEngine, error) { + var definition model.Definition + var err error + + if len(uri) > 0 { + definition, err = model.LoadPipelineDefinitionFromFile(uri) + if err != nil { + return MaskEngine{nil}, err + } + // merge the current seed with the seed provided by configuration on the pipe + definition.Seed += seed + } + + pipeline := model.NewPipeline(nil) + pipeline, _, err = model.BuildPipeline(pipeline, definition, caches, fns, "", "") + + return MaskEngine{pipeline}, err +} + +func (me MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Entry, error) { + log.Info().Msg("Mask apply") + + var result []model.Entry + + err := me.pipeline. + WithSource(model.NewSourceFromSlice([]model.Dictionary{model.NewDictionary().With(".", e)})). + // Process(model.NewCounterProcessWithCallback("internal", 1, updateContext)). + AddSink(model.NewSinkToSlice(&result)). + Run() + if err != nil { + return nil, err + } + + return result[0], nil +} + +// Factory create a mask from a configuration +func Factory(conf model.MaskFactoryConfiguration) (model.MaskEngine, bool, error) { + if len(conf.Masking.Mask.Apply.URI) > 0 { + // set differents seeds for differents jsonpath + h := fnv.New64a() + h.Write([]byte(conf.Masking.Selector.Jsonpath)) + conf.Seed += int64(h.Sum64()) //nolint:gosec + mask, err := NewMask(conf.Seed, conf.Cache, conf.Functions, conf.Masking.Mask.Apply.URI) + if err != nil { + return mask, true, err + } + return mask, true, nil + } + return nil, false, nil +} diff --git a/pkg/model/model.go b/pkg/model/model.go index 036c64eb..aa7f96ae 100755 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -237,6 +237,10 @@ type Sha3Type struct { MaxStrLen int `yaml:"maxstrlen,omitempty" json:"maxstrlen,omitempty" jsonschema_description:"an error will occur if the identifier can grow longer than the specified length"` } +type ApplyType struct { + URI string `yaml:"uri" json:"uri" jsonschema_description:"URI of the mask resource"` +} + type MaskType struct { Add Entry `yaml:"add,omitempty" json:"add,omitempty" jsonschema:"oneof_required=Add,title=Add Mask,description=Add a new field in the JSON stream"` AddTransient Entry `yaml:"add-transient,omitempty" json:"add-transient,omitempty" jsonschema:"oneof_required=AddTransient,title=Add Transient Mask" jsonschema_description:"Add a new temporary field, that will not show in the JSON output"` @@ -275,6 +279,7 @@ type MaskType struct { TimeLine TimeLineType `yaml:"timeline,omitempty" json:"timeline,omitempty" jsonschema:"oneof_required=TimeLine,title=TimeLine Mask" jsonschema_description:"Generate a timeline under constraints and rules"` Sequence SequenceType `yaml:"sequence,omitempty" json:"sequence,omitempty" jsonschema:"oneof_required=Sequence,title=Sequence Mask" jsonschema_description:"Generate a sequenced ID that follows specified format"` Sha3 Sha3Type `yaml:"sha3,omitempty" json:"sha3,omitempty" jsonschema:"oneof_required=Sha3,title=Sha3 Mask" jsonschema_description:"Generate a variable-length crytographic hash (collision resistant)"` + Apply ApplyType `yaml:"apply,omitempty" json:"apply,omitempty" jsonschema:""` } type Masking struct { diff --git a/schema/v1/pimo.schema.json b/schema/v1/pimo.schema.json index 1cfe683b..46e85f46 100644 --- a/schema/v1/pimo.schema.json +++ b/schema/v1/pimo.schema.json @@ -3,6 +3,19 @@ "$id": "https://github.com/cgi-fr/pimo/pkg/model/definition", "$ref": "#/$defs/Definition", "$defs": { + "ApplyType": { + "properties": { + "uri": { + "type": "string", + "description": "URI of the mask resource" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "uri" + ] + }, "CacheDefinition": { "properties": { "unique": { @@ -754,6 +767,9 @@ "$ref": "#/$defs/Sha3Type", "title": "Sha3 Mask", "description": "Generate a variable-length crytographic hash (collision resistant)" + }, + "apply": { + "$ref": "#/$defs/ApplyType" } }, "additionalProperties": false,