Task/rdmp 32 regex redaction (#2009)

* interim

* interim

* redact and add some tests

* add missing interface

* add regex redaction keys

* add helper

* add concept

* update interfaces

* add icons and fix restore

* add missing files

* add docs

* add regions

* add ignore

* add speed test

* improve speed

* improve speed

* update padding

* stress test

* add test case

* add fk names

* faster data load

* faster update

* more timers

* super fast redaction

* tidy up code

* woking keys

* improved

* add limiting

* shared helper

* shared

* interim

* working

* add summary

* restore tests working

* add tests

* start og ui update

* start of ui

* basic configuraion

* working add

* user interface

* update for tests

* update test

* tests

* tidy up

* updates

* fix merge

* ad dicons

* add missing icons

* tidyup

* fix tests

* add missing file

* update migration number

* add changelog

* add timeout

* fix build

* interim

* tidy up

* tidy up code

* add publish

* add todo

* add redact all

* add column

* update table

* add docs

* update markdown

* fix typo

* fix typos

* tidy up tests

* tidy up tests

* Include regex redaction doc in VS Solution file

* Update ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs

Syntax cleanup

* Update ThrowImmediatelyActivator.cs

Remove obsolete comment

* Update ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs

Start being green and recycling

* Update ExecuteCommandPerformRegexRedactionOnCatalogueTests.cs

Finish recycling test objects

---------

Co-authored-by: James A Sutherland <[email protected]>

Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.cs

@@ -51,11 +51,11 @@ public RDMPTaskBarUI()
         btnTables.Image = CatalogueIcons.TableInfo.ImageToBitmap();
         btnTables.BackgroundImage = BackColorProvider.GetBackgroundImage(btnTables.Size, RDMPCollection.Tables);
 
-        btnDataSets.Image = CatalogueIcons.Dataset.ImageToBitmap();
-        btnDataSets.BackgroundImage = BackColorProvider.GetBackgroundImage(btnDataSets.Size, RDMPCollection.Datasets);
+        btnConfiguration.Image = FamFamFamIcons.pencil_go.ImageToBitmap();
+        btnConfiguration.BackgroundImage = BackColorProvider.GetBackgroundImage(btnConfiguration.Size, RDMPCollection.Datasets);
 
         btnLoads.Image = CatalogueIcons.LoadMetadata.ImageToBitmap();
-        btnLoads.BackgroundImage = BackColorProvider.GetBackgroundImage(btnDataSets.Size, RDMPCollection.DataLoad);
+        btnLoads.BackgroundImage = BackColorProvider.GetBackgroundImage(btnConfiguration.Size, RDMPCollection.DataLoad);
 
         btnFavourites.Image = CatalogueIcons.Favourite.ImageToBitmap();
         btnDeleteLayout.Image = FamFamFamIcons.delete.ImageToBitmap();
@@ -111,7 +111,7 @@ private void SetupToolTipText()
             btnTables.ToolTipText = "Advanced features e.g. logging, credentials, dashboards etc";
             btnLoads.ToolTipText = "Load configurations for reading data into your databases";
             btnFavourites.ToolTipText = "Collection of all objects that you have favourited";
-            btnDataSets.ToolTipText = "All external datasets that have been configured for use in RDMP";
+            btnConfiguration.ToolTipText = "All external datasets that have been configured for use in RDMP";
         }
         catch (Exception e)
         {
@@ -190,8 +190,8 @@ private RDMPCollection ButtonToEnum(object button)
             collectionToToggle = RDMPCollection.SavedCohorts;
         else if (button == btnFavourites)
             collectionToToggle = RDMPCollection.Favourites;
-        else if (button == btnDataSets)
-            collectionToToggle = RDMPCollection.Datasets;
+        else if (button == btnConfiguration)
+            collectionToToggle = RDMPCollection.Configurations;
         else
             throw new ArgumentOutOfRangeException(nameof(button));
 

Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.resx

@@ -178,7 +178,7 @@
         TgDQASA1MVpwzwAAAABJRU5ErkJggg==
 </value>
   </data>
-  <data name="btnDataSets.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+  <data name="btnConfiguration.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
     <value>
         iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8
         YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAIDSURBVDhPpZLrS5NhGMb3j4SWh0oRQVExD4gonkDpg4hG

Application/ResearchDataManagementPlatform/WindowManagement/WindowManager.cs

@@ -148,10 +148,10 @@ public PersistableToolboxDockContent Create(RDMPCollection collectionToCreate,
                     Image.Load<Rgba32>(CatalogueIcons.Favourite));
                 break;
 
-            case RDMPCollection.Datasets:
-                collection = new DatasetsCollectionUI();
-                toReturn = Show(RDMPCollection.Datasets, collection, "Datasets",
-                    Image.Load<Rgba32>(CatalogueIcons.Dataset));
+            case RDMPCollection.Configurations:
+                collection = new ConfigurationsCollectionUI();
+                toReturn = Show(RDMPCollection.Configurations, collection, "Configurations",
+                     Image.Load<Rgba32>(FamFamFamIcons.pencil_go));
                 break;
 
             default: throw new ArgumentOutOfRangeException(nameof(collectionToCreate));

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [8.4.0] - Unreleased
 
 - Add Ordering to Filters
+- [MSSQL ONLY] Add ability to perform Regex redactions on data loads and existing catalogues
 - Add overview page for Catalogues
 - Add RAW Table Date Column Override for Delta Loads
 - Fix Delta Load off by one issue

Documentation/DataLoadEngine/RegexRedactions.md

@@ -0,0 +1,44 @@
+# Regex Redactions
+
+RDMP has the ability to redact incoming and existing catalogue data.
+It uses user-defined regex to do this.
+This functionality may be useful if you know about problematic data that you wish to clean before it reaches RDMP
+
+## Setup & Configuration
+To begin, a redaction configuration is required.
+This can be added in the "Configurations" section of RDMP.
+Each configuration has:
+* A Name
+* A Description
+* A Regex string to match
+* A Replacement value
+
+N.B. The replacement value cannot be longer than the redacted string i.e. "string" => "myRedactedString" will not work.
+
+Once the configuration has been saved, it can be used to redact data.
+
+## Usage in Data Loads
+Within the Data Loads mutilators, there is a mutilator named "Regex Redaction Mutilator" that can be used in the RAW or STAGING areas of the load.
+This mutilator will redact based on your selected configuration.
+The configuration for this mutilator requires:
+* A predefined regex redaction configuration (see above)
+* Either a regex to select which columns to redact, or a selection of known columns
+
+N.B. Primary Key columns will not be redacted and this functionality is intended for string fields.
+
+Once the configuration has been set, it will redact the desired columns during a data load.
+
+During a run, the mutilator will replace any regex matches with the desired string and store the now known redaction in RDMP.
+To view the redactions made, see the section below.
+
+
+## Usage within RDMP
+You can view and restore existing redactions, along with adding new redactions within RDMP.
+For a Catalogue, right-click and open the "Catalogue Items" submenu. 
+From here, select the "Regex Redactions" option.
+This will open an interface displaying existing redactions.
+From here you can restore existing redactions either individually or in bulk.
+You can also apply regex redactions to existing data by selecting a redaction configuration and which columns to redact.
+
+
+[Catalogue](../CodeTutorials/Glossary.md#Catalogue)

HIC.DataManagementPlatform.sln

@@ -10,8 +10,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		CHANGELOG.md = CHANGELOG.md
 		deadlinksconfig.json = deadlinksconfig.json
 		Directory.Build.props = Directory.Build.props
-		.github\workflows\links.yml = .github\workflows\links.yml
 		Directory.Packages.props = Directory.Packages.props
+		.github\workflows\links.yml = .github\workflows\links.yml
 		NoteForNewDevelopers.md = NoteForNewDevelopers.md
 		Documentation\CodeTutorials\Packages.md = Documentation\CodeTutorials\Packages.md
 		rdmp-client.xml = rdmp-client.xml
@@ -55,6 +55,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Documentation", "Documentat
 		Documentation\CodeTutorials\Packages.md = Documentation\CodeTutorials\Packages.md
 		Documentation\CodeTutorials\PluginWriting.md = Documentation\CodeTutorials\PluginWriting.md
 		Documentation\CodeTutorials\RdmpCommandLine.md = Documentation\CodeTutorials\RdmpCommandLine.md
+		Documentation\DataLoadEngine\RegexRedactions.md = Documentation\DataLoadEngine\RegexRedactions.md
 		Documentation\CodeTutorials\Tests.md = Documentation\CodeTutorials\Tests.md
 		Documentation\CodeTutorials\UserInterfaceOverview.md = Documentation\CodeTutorials\UserInterfaceOverview.md
 		Documentation\CodeTutorials\UserManual.md = Documentation\CodeTutorials\UserManual.md
@@ -315,4 +316,4 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
-EndGlobal
+EndGlobal