feat: preserve list (#321)

* feat: preserve list

* test: preserve list

* docs: preserve-list

CHANGELOG.md

@@ -14,6 +14,10 @@ Types of changes
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [1.26.0]
+
+- `Added` property `preserve-list` to masking definition to be able to ignore masking specific values
+
 ## [1.25.0]
 
 - `Added` mask `sha3` to generate collision resistant identifiers

README.md

@@ -51,6 +51,7 @@ masking:
     mask:
       type: "argument"
     preserve: "null"
+    preserve-list: ["value to preserve"]
 
 caches:
   cacheName:
@@ -68,7 +69,9 @@ caches:
 `jsonpath` defines the path of the entry that has to be masked in the json file.
 `mask` defines the mask that will be used for the entry defined by `selector`.
 `cache` is optional, if the current entry is already in the cache as key the associated value is returned without executing the mask. Otherwise the mask is executed and a new entry is added in the cache with the orignal content as `key` and the masked result as `value`. The cache have to be declared in the `caches` section of the YAML file.
-`preserve` is optional, and is used to keep some values unmasked in the json file. Allowed `preserve` options are: `"null"` (null values), `"empty"` (empty string `""`), and `"blank"` (both `empty` and `null` values). Additionally, `preserve` can be used with mask [`fromCache`](#fromCache) to preserve uncached values. (usage: `preserve: "notInCache"`)
+`preserve` is optional, and is used to keep some values unmasked in the json file. Allowed `preserve` options are: `"null"` (null values), `"empty"` (empty string `""`), and
+`"blank"` (both `empty` and `null` values). Additionally, `preserve` can be used with mask [`fromCache`](#fromCache) to preserve uncached values. (usage: `preserve: "notInCache"`)
+`preserve-list` is optional and is used to ignore specific values
 
 Multiple masks can be applied on the same jsonpath location, like in this example :
 

pkg/model/cache_test.go

@@ -81,7 +81,7 @@ func TestFromCacheProcessShouldWaitForValueProvide(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("id"), NewMaskCacheEngine(cache, idMasking), "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("id"), NewMaskCacheEngine(cache, idMasking), "", nil, "")).
 		Process(NewFromCacheProcess(NewPathSelector("supervisor"), cache, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
@@ -110,7 +110,7 @@ func TestFromCacheProcessShouldWaitForLoopProvid(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("id"), NewMaskCacheEngine(cache, idMasking), "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("id"), NewMaskCacheEngine(cache, idMasking), "", nil, "")).
 		Process(NewFromCacheProcess(NewPathSelector("supervisor"), cache, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
@@ -142,7 +142,7 @@ func TestFromCacheProcessShouldUsedPreviouslyCachedValue(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("id"), NewMaskCacheEngine(cache, idMasking), "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("id"), NewMaskCacheEngine(cache, idMasking), "", nil, "")).
 		Process(NewFromCacheProcess(NewPathSelector("supervisor"), cache, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
@@ -174,7 +174,7 @@ func TestFromCacheProcessShouldReorderList(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("id"), NewMaskCacheEngine(cache, idMasking), "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("id"), NewMaskCacheEngine(cache, idMasking), "", nil, "")).
 		Process(NewFromCacheProcess(NewPathSelector("supervisor"), cache, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
@@ -206,7 +206,7 @@ func TestFromCacheProcessShouldWaitWithUnique(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("id"), NewUniqueMaskCacheEngine(cache, idMasking), "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("id"), NewUniqueMaskCacheEngine(cache, idMasking), "", nil, "")).
 		Process(NewFromCacheProcess(NewPathSelector("supervisor"), cache, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()

pkg/model/model.go

@@ -288,6 +288,7 @@ type Masking struct {
 	Masks     []MaskType     `yaml:"masks,omitempty" json:"masks,omitempty" jsonschema:"oneof_required=case2,oneof_required=case4" jsonschema_description:"Defines how the selected value(s) will be masked"`
 	Cache     string         `yaml:"cache,omitempty" json:"cache,omitempty" jsonschema_description:"Use an in-memory cache to preserve coherence between original/masked values"`
 	Preserve  string         `yaml:"preserve,omitempty" json:"preserve,omitempty" jsonschema:"enum=null,enum=empty,enum=blank,enum=notInCache" jsonschema_description:"Preserve (do not mask) some values : null = preserve null value, empty = preserve empty strings, blank = preserve both null and empty values, notInCache = preserve value even if not present in cache (fromCache mask)"`
+	PreserveL []Entry        `yaml:"preserve-list,omitempty" json:"preserve-list,omitempty" jsonschema_description:"Preserve (do not mask) given values"`
 	Seed      SeedType       `yaml:"seed,omitempty" json:"seed,omitempty" jsonschema_description:"Initialize the Pseaudo-Random-Generator with the value given field"`
 }
 

pkg/model/model_test.go

@@ -172,7 +172,7 @@ func TestPipelineWithMaskEngine(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("name"), nameMasking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("name"), nameMasking, "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 
@@ -208,7 +208,7 @@ func TestMaskEngineShouldNotCreateField(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("name"), nameMasking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("name"), nameMasking, "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 
@@ -255,7 +255,7 @@ func TestMaskEngineShouldMaskAllEntriesInArray(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("city"), nameMasking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("city"), nameMasking, "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 
@@ -272,7 +272,7 @@ func TestMaskEngineShouldMaskNestedEntry(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("address.city"), nameMasking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("address.city"), nameMasking, "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 
@@ -296,7 +296,7 @@ func TestMaskEngineShouldMaskNestedDictionariesArray(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("address.city"), nameMasking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("address.city"), nameMasking, "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 
@@ -326,7 +326,7 @@ func TestMaskEngineShouldMaskNestedArray(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("address.city"), nameMasking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("address.city"), nameMasking, "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 
@@ -399,7 +399,7 @@ func TestMaskEngineShouldMaskNestedArrays(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("persons.phonenumber"), nameMasking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("persons.phonenumber"), nameMasking, "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 
@@ -422,7 +422,7 @@ func TestMaskEngineShouldMaskNestedNestedArrays(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("elements.persons.phonenumber"), nameMasking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("elements.persons.phonenumber"), nameMasking, "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 
@@ -439,7 +439,7 @@ func TestInOutFormat1(t *testing.T) {
 	iput := `{"key": ["mask"]}`
 	idict := jsonlineToDictionaries(iput)
 	pipeline := NewPipelineFromSlice(idict).
-		Process(NewMaskEngineProcess(NewPathSelector("key"), masking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("key"), masking, "", nil, "")).
 		AddSink(NewSinkToSlice(&odict))
 	err := pipeline.Run()
 
@@ -453,7 +453,7 @@ func TestInOutFormat2(t *testing.T) {
 	iput := `{"key1": [{"key2": "mask"}]}`
 	idict := jsonlineToDictionaries(iput)
 	pipeline := NewPipelineFromSlice(idict).
-		Process(NewMaskEngineProcess(NewPathSelector("key1.key2"), masking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("key1.key2"), masking, "", nil, "")).
 		AddSink(NewSinkToSlice(&odict))
 	err := pipeline.Run()
 
@@ -478,8 +478,8 @@ func TestMaskEngineShouldMaskMultipleNestedNestedArrays(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("elements.persons.email"), emailMasking, "", "")).
-		Process(NewMaskEngineProcess(NewPathSelector("elements.persons.phonenumber"), nameMasking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("elements.persons.email"), emailMasking, "", nil, "")).
+		Process(NewMaskEngineProcess(NewPathSelector("elements.persons.phonenumber"), nameMasking, "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 
@@ -497,7 +497,7 @@ func TestMaskEngineShouldReturnError(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("city"), errorMasking, "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("city"), errorMasking, "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 
@@ -566,7 +566,7 @@ func TestCacheShouldProvide(t *testing.T) {
 	var result []Entry
 
 	pipeline := NewPipelineFromSlice(mySlice).
-		Process(NewMaskEngineProcess(NewPathSelector("city"), NewMaskCacheEngine(cache, errorMasking), "", "")).
+		Process(NewMaskEngineProcess(NewPathSelector("city"), NewMaskCacheEngine(cache, errorMasking), "", nil, "")).
 		AddSink(NewSinkToSlice(&result))
 	err := pipeline.Run()
 

pkg/model/pipeline.go

@@ -122,6 +122,7 @@ func BuildPipeline(pipeline Pipeline, conf Definition, caches map[string]Cache,
 					Masks:     nil,
 					Cache:     masking.Cache,
 					Preserve:  masking.Preserve,
+					PreserveL: masking.PreserveL,
 					Seed:      masking.Seed,
 				}
 
@@ -154,7 +155,7 @@ func BuildPipeline(pipeline Pipeline, conf Definition, caches map[string]Cache,
 								mask = NewMaskCacheEngine(typedCache, mask)
 							}
 						}
-						pipeline = pipeline.Process(NewMaskEngineProcess(NewPackedPathSelector(virtualMask.Selector.Jsonpath), mask, virtualMask.Preserve, skipLogFile))
+						pipeline = pipeline.Process(NewMaskEngineProcess(NewPackedPathSelector(virtualMask.Selector.Jsonpath), mask, virtualMask.Preserve, virtualMask.PreserveL, skipLogFile))
 						nbArg++
 					}
 				}

pkg/model/process_mask.go

@@ -21,21 +21,23 @@ import (
 	over "github.com/adrienaury/zeromdc"
 	"github.com/cgi-fr/pimo/pkg/statistics"
 	"github.com/rs/zerolog/log"
+	"golang.org/x/exp/slices"
 )
 
-func NewMaskEngineProcess(selector Selector, mask MaskEngine, preserve string, skipLogFile string) Processor {
+func NewMaskEngineProcess(selector Selector, mask MaskEngine, preserve string, preserveList []Entry, skipLogFile string) Processor {
 	var errlogger *MsgLogger
 	if len(skipLogFile) > 0 {
 		errlogger = NewMsgLogger(skipLogFile)
 	}
-	return &MaskEngineProcess{selector, mask, preserve, errlogger}
+	return &MaskEngineProcess{selector, mask, preserve, preserveList, errlogger}
 }
 
 type MaskEngineProcess struct {
-	selector  Selector
-	mask      MaskEngine
-	preserve  string
-	errlogger *MsgLogger
+	selector     Selector
+	mask         MaskEngine
+	preserve     string
+	preserveList []Entry
+	errlogger    *MsgLogger
 }
 
 func (mep *MaskEngineProcess) Open() error {
@@ -49,6 +51,9 @@ func (mep *MaskEngineProcess) ProcessDictionary(dictionary Dictionary, out Colle
 	result := dictionary
 	applied := mep.selector.Apply(result, func(rootContext, parentContext Dictionary, key string, value Entry) (Action, Entry) {
 		switch {
+		case slices.Contains(mep.preserveList, value):
+			log.Trace().Msgf("Preserve specific value, skip masking")
+			return NOTHING, nil
 		case value == nil && (mep.preserve == "null" || mep.preserve == "blank"):
 			log.Trace().Msgf("Preserve %s value, skip masking", mep.preserve)
 			return NOTHING, nil

schema/v1/pimo.schema.json

@@ -823,6 +823,11 @@
           ],
           "description": "Preserve (do not mask) some values : null = preserve null value, empty = preserve empty strings, blank = preserve both null and empty values, notInCache = preserve value even if not present in cache (fromCache mask)"
         },
+        "preserve-list": {
+          "items": true,
+          "type": "array",
+          "description": "Preserve (do not mask) given values"
+        },
         "seed": {
           "$ref": "#/$defs/SeedType",
           "description": "Initialize the Pseaudo-Random-Generator with the value given field"

test/suites/masking_preserve.yml

@@ -123,6 +123,38 @@ testcases:
   - script: rm -f expected.txt
   - script: rm -f output.txt
 
+- name: preserve specific values
+  steps:
+  - script: rm -f masking.yml
+  - script: |-
+      cat > expected.txt <<EOF
+      {"name":"maskedName"}
+      {"name":"toPreserve"}
+      {"name":""}
+      {"name":null}
+      EOF
+  - script: |-
+      cat > masking.yml <<EOF
+      version: "1"
+      masking:
+        - selector:
+            jsonpath: "name"
+          preserve-list: ["toPreserve", "", null]
+          mask:
+            constant: "maskedName"
+      EOF
+  - script: |-
+      echo -e '{"name":"paul"}\n{"name":"toPreserve"}\n{"name":""}\n{"name":null}' | pimo > output.txt
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemerr ShouldBeEmpty
+  - script: |-
+      diff expected.txt output.txt
+    assertions:
+    - result.systemout ShouldBeEmpty
+  - script: rm -f expected.txt
+  - script: rm -f output.txt
+
 - name: preserve notInCache values
   steps:
   - script: rm -f masking.yml