Release/8.2.0

.github/workflows/build.yml

@@ -179,15 +179,14 @@ jobs:
           for plugin in https://api.github.com/repos/SMI/RdmpDicom/releases/latest https://api.github.com/repos/HICServices/HicPlugin/releases/latest https://api.github.com/repos/HICServices/RdmpExtensions/releases/latest
           do
             PluginName="$(cut -d/ -f6 <<< $plugin)"
-            NAME="$(curl -s $plugin | grep "browser_download_url.*$PluginName.*nupkg" | cut -d : -f 2,3 | cut -d "\"" -f 2)"
+            NAME="$(curl -s $plugin | grep "browser_download_url.*$PluginName.*rdmp" | cut -d : -f 2,3 | cut -d "\"" -f 2)"
             curl -OL  $NAME
           done
-          ls *.nupkg > rdmpplugins.txt
           for platform in PublishWindows PublishLinux PublishWinForms
           do
-            cp rdmpplugins.txt *.nupkg $platform
+            cp *.rdmp $platform
           done
-          rm rdmpplugins.txt *.nupkg
+          rm *.rdmp
       - name: Sign & zip
         shell: bash
         run: |

Application/ResearchDataManagementPlatform/Menus/RDMPTopMenuStripUI.cs

@@ -407,6 +407,12 @@ private void userSettingsToolStripMenuItem_Click(object sender, EventArgs e)
         settings.Show();
     }
 
+    private void instanceSettingsToolStripMenuItem_Click(object sender, EventArgs e)
+    {
+        var settings = new InstanceSettings(Activator);
+        settings.Show();
+    }
+
     private void licenseToolStripMenuItem_Click(object sender, EventArgs e)
     {
         var l = new LicenseUI();
@@ -457,6 +463,8 @@ private void navigateForwardToolStripMenuItem_Click(object sender, EventArgs e)
         _windowManager.Navigation.Forward(true);
     }
 
+
+
     private void checkForUpdatesToolStripMenuItem_Click(object sender, EventArgs e)
     {
         // AutoUpdater.NET is Windows-only for now:

Application/ResearchDataManagementPlatform/Menus/RDMPTopMenuStripUI.resx

@@ -1,4 +1,64 @@
-<root>
+<?xml version="1.0" encoding="utf-8"?>
+<root>
+  <!--
+    Microsoft ResX Schema 
+
+    Version 2.0
+
+    The primary goals of this format is to allow a simple XML format
+    that is mostly human readable. The generation and parsing of the
+    various data types are done through the TypeConverter classes
+    associated with the data types.
+
+    Example:
+
+    ... ado.net/XML headers & schema ...
+    <resheader name="resmimetype">text/microsoft-resx</resheader>
+    <resheader name="version">2.0</resheader>
+    <resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
+    <resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
+    <data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
+    <data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
+    <data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
+        <value>[base64 mime encoded serialized .NET Framework object]</value>
+    </data>
+    <data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+        <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
+        <comment>This is a comment</comment>
+    </data>
+
+    There are any number of "resheader" rows that contain simple
+    name/value pairs.
+
+    Each data row contains a name, and value. The row also contains a
+    type or mimetype. Type corresponds to a .NET class that support
+    text/value conversion through the TypeConverter architecture.
+    Classes that don't support this are serialized and stored with the
+    mimetype set.
+
+    The mimetype is used for serialized objects, and tells the
+    ResXResourceReader how to depersist the object. This is currently not
+    extensible. For a given mimetype the value must be set accordingly:
+
+    Note - application/x-microsoft.net.object.binary.base64 is the format
+    that the ResXResourceWriter will generate, however the reader can
+    read any of the formats listed below.
+
+    mimetype: application/x-microsoft.net.object.binary.base64
+    value   : The object must be serialized with
+            : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
+            : and then encoded with base64 encoding.
+
+    mimetype: application/x-microsoft.net.object.soap.base64
+    value   : The object must be serialized with
+            : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
+            : and then encoded with base64 encoding.
+
+    mimetype: application/x-microsoft.net.object.bytearray.base64
+    value   : The object must be serialized into a byte array
+            : using a System.ComponentModel.TypeConverter
+            : and then encoded with base64 encoding.
+    -->
   <xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
     <xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
     <xsd:element name="root" msdata:IsDataSet="true">
@@ -96,7 +156,7 @@
   <data name="viewHistoryToolStripMenuItem.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
     <value>
         iVBORw0KGgoAAAANSUhEUgAAABMAAAATCAYAAAByUDbMAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
-        wQAADsEBuJFr7QAAAO5JREFUOE/lk7ENwjAQRSkpKRmBMSgpKBmAERiBHqQMwBAMQUFBkRkQEi0lBQX8
+        wAAADsABataJCQAAAO5JREFUOE/lk7ENwjAQRSkpKRmBMSgpKBmAERiBHqQMwBAMQUFBkRkQEi0lBQX8
         F/wVy4klhxZ9PeUO537O5jyqWs3FSTzEOeSD5IDCdw9LUSwHdETxTkxFFfKLKJYDtkYxRogn+TPkRXLA
         GfV1VotiOeBsKE5ZiWJFYWN4Fzbai0GKwkZbYbOX+LkzZDP/u7ARqXpnsl3+ymY8MbHhQUwEys6kTazY
         DK0F48FvbJturiHvzKRNrNQMzcRR2NR0ZtIFiPb91VvIY43FQuQ6q/1i9hxEquxM+oWhdxND1tg6t6QZ

CHANGELOG.md

@@ -1,10 +1,25 @@
-
 # Changelog
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [8.2.0] - Unreleased
+
+## Changed
+
+- Add Key-Value store for instance settings
+- Allow for Re-extractions of projects to a database, see [ExecuteFullExtractionToDatabaseMSSql](Documentation\DataExtractions\ExecuteFullExtractionToDatabaseMSSql.md)
+- Add ability to use .rdmp plugin files
+- Add the ability to store versions of cohort configurations, see [Cohort Versioning](Documentation\Cohorts\CohortVersioning.md)
+- Add ability to restrict GROUPBY clause in cohort aggregate builder
+- When cloning an ExtractionConfiguration with a deprecated catalogue, the GUI will ask if you want to replace the deprecated catalogue with the known replacement
+- Add ability to customise LoadMetdata Folder Location. See [LoadMetadata](Documentation\DataLoadEngine\LoadMetadata.md)
+- Add ability to point a catalogue to a new data source [Documentation](./Documentation/Catalogues/UpdateCatalogueDataLocation.md)
+- Allow DQE graphs to be scrollable and scalable
+- Allow for partial refreshes of time-based DQE charts
+- Fix issue when creating PostgreSQL Logging Server
+
 ## [8.1.7] - 2024-06-17
 
 ## Changed
@@ -59,7 +74,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Bump HIC.BadMedicine from 1.2.0 to 1.2.1 
 - Bump NPOI from 2.6.2 to 2.7.0 
 
-
 ## [8.1.4] - 2024-02-19
 
 ## Changed

Documentation/Catalogues/UpdateCatalogueDataLocation.md

@@ -0,0 +1,17 @@
+# Update A [Catalogue](../CodeTutorials/Glossary.md#Catalogue) Data Location
+It can be useful to migrate data from one database to another.
+Instead of reimporting a [catalogue](../CodeTutorials/Glossary.md#Catalogue), RDMP allows you to repoint the [catalogue](../CodeTutorials/Glossary.md#Catalogue) to the location of your moved data.
+
+## How to
+* Right Click an a Catalogue > Catalogue Items > Update Catalogue Data Location
+* This will open a new dialog
+* From here, you can select which columns you wish to update and tell RDMP where the new location is
+* RDMP will perform several checks to make sure the new data location is available and of the correct types
+* It will inform you of any issues it comes across, otherwise will migrate the catalogue data references to the selected location
+
+## What Does it change?
+This functionality updates the underlying [ColumnInfo](../CodeTutorials/Glossary.md#ColumnInfo) to point to the new table. It also updates the extractionInformation SelectSQL to allow for future extractions to continue to work.
+If you select a never-before-seen table, RDMP will also generate a new known table record in case of future use
+
+[Catalogue]: ../CodeTutorials/Glossary.md#Catalogue
+[ColumnInfo]: ../CodeTutorials/Glossary.md#ColumnInfo

Documentation/CodeTutorials/FAQ.md

@@ -412,7 +412,7 @@ Yes, [determining database types from untyped data (e.g. CSV)](./DataTableUpload
 
 <a name="dle-folders"></a>
 ### What is the purpose of the the data load folders (ForLoading, ForArchiving etc)?
-RDMP data load jobs are configured by creating a [LoadMetadata].  When you create a new [LoadMetadata] you will be prompted to choose/create a set of load folders.  The current value is stored in the `LocationOfFlatFiles` field of [LoadMetadata].  The directories created have the following layout:
+RDMP data load jobs are configured by creating a [LoadMetadata].  When you create a new [LoadMetadata] you will be prompted to choose/create a set of load folders.  The default directories created have the following layout:
 
 ```
 someFolder

Documentation/CodeTutorials/TestPlan.md

@@ -53,4 +53,4 @@ It may be useful at this point to revisit each piece of functionality and ensure
 * Does the release work as epxected with a fresh install and via the upgrade path?
 * Is all functionality documented?
 * Are all version numbers bumped correctly?
-* Do all of the managed plugins work with the new release without issues or warnings?
+* Do all of the managed plugins work with the new release without issues or warnings?

Documentation/Cohorts/CohortVersioning.md

@@ -0,0 +1,26 @@
+# Cohort Versioning
+Cohort Identity Configurations can be versioned, allowing you to store copies of cohort configurations prior to making updates.
+you may want to use versioning instead of cloning the cohort to keep the tree list of cohort configurations simple, as versions do not appear in the traditional RDMP tree structure.
+
+Cohort versioning is accessible via the Cohort creation page within RDMP, from here you can open existing versions of the cohort, or save the current configuration as a new version.
+
+## Command Line
+Versioning can be performed by using the CreateVersionOfCohortConfiguration command.
+It can be ran with the following options 
+```
+CreateVersionOfCohortConfiguration cic:{some_id} name:{some_name}
+```
+
+Cohort versions can be used in the same way as a standard cohort identification configuration, their ID is accessible via the command
+```
+ListCohortVersions cic:{some_id}
+```
+This command will list the names and IDs of and versions associated with a cohort configuration
+
+Within the CLI, cohort versions will appear alongside top-level cohort identity configurations when performing commands such as
+```
+List cic:*
+```
+
+## Cloning a cohort with Versions
+Cloning a cohort configuration with versions does not clone the versions alongside the cohort configuration

Documentation/DataExtractions/ExecuteFullExtractionToDatabaseMSSql.md

@@ -0,0 +1,33 @@
+## Execute Full Extraction To Database MSSQL
+This data extraction pipeline component allows you to extract data into a known database.
+You can add the database you want to extract to as an external database within RDMP.
+
+This component has several configurable options that are detailed below
+| Configuration Option                            | Description                                                                                                                                                                                                                                                                                                                                                                   |
+|-------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Target Database Server                          | External server to create the extraction into, a new database will be created for the [Project]: ../CodeTutorials/Glossary.md#Project based on the naming pattern provided                                                                                                                                                                                                                                            |
+| Database Naming Pattern                         | How do you want to name datasets, use the following tokens if you need them:             $p - [Project](../CodeTutorials/Glossary.md#Project) Name ('e.g. My [Project](../CodeTutorials/Glossary.md#Project)')          $n - [Project](../CodeTutorials/Glossary.md#Project) Number (e.g. 234)          $t - Master Ticket (e.g. 'LINK-1234')          $r - Request Ticket (e.g. 'LINK-1234')          $l - Release Ticket (e.g. 'LINK-1234')  Default : Proj_$n_$l                                   |
+| Table Naming Pattern                            | How do you want to name datasets, use the following tokens if you need them:             $p - [Project](../CodeTutorials/Glossary.md#Project) Name ('e.g. My [Project](../CodeTutorials/Glossary.md#Project)')          $n - [Project](../CodeTutorials/Glossary.md#Project) Number (e.g. 234)          $c - Configuration Name (e.g. 'Cases')          $d - Dataset name (e.g. 'Prescribing')          $a - Dataset acronym (e.g. 'Presc')            You must have either $a or $d  Default : $c_$d |
+| Drop Table If Load Fails                        | If the extraction fails half way through AND the destination table was created during the extraction then the table will be dropped from the destination rather than being left in a half loaded state                                                                                                                                                                        |
+| Alter Timeout                                   | Timeout to perform all ALTER TABLE operations (column resize and PK creation)                                                                                                                                                                                                                                                                                                 |
+| Copy Collations                                 | True to copy the column collations from the source database when creating the destination database.  Only works if both the source and destination have the same DatabaseType.  Excludes columns which feature a transform as part of extraction.                                                                                                                             |
+| Always Drop ExtractionTables                    | True to always drop the destination database table(s) from the destination if they already existed                                                                                                                                                                                                                                                                            |
+| Make Final Table Distinct When Batching Results | True to apply a distincting operation to the final table when using an [ExtractionProgress](../CodeTutorials/Glossary.md#ExtractionProgress).  This prevents data duplication from failed batch resumes.                                                                                                                                                                                                                         |
+| Append Data If Table Exists                     | If this extraction has already been run, it will append the extraction data into the database. There is no duplication protection with this functionality.                                                                                                                                                                                                                    |
+| Include Timestamp                               | If checked, a column names 'extraction_timestamp' will be included in the extraction that denotes the time the record was added to the extraction.                                                                                                                                                                                                                            |
+| Use Acronym For File Naming                     | Naming of flat files is usually based on [Catalogue](../CodeTutorials/Glossary.md#Catalogue).Name, if this is true then the [Catalogue](../CodeTutorials/Glossary.md#Catalogue).Acronym will be used instead                                                                                                                                                                                                                                                      |
+| Date Format                                     | The date format to output all datetime fields in e.g. dd/MM/yyyy for uk format yyyy-MM-dd for something more machine processable, see https://msdn.microsoft.com/en-us/library/8kb3ddd4(v=vs.110).aspx                                                                                                                                                                        |
+| Clean Extraction Folder Before Extraction       | If this is true, the dataset/globals extraction folder will be wiped clean before extracting the dataset. Useful if you suspect there are spurious files in the folder                                                                                                                                                                                                        |
+| Extraction Subdirectory Pattern                 | Overrides the extraction sub directory of datasets as they are extracted          $c - Configuration Name (e.g. 'Cases')          $i - Configuration ID (e.g. 459)          $d - Dataset name (e.g. 'Prescribing')          $a - Dataset acronym (e.g. 'Presc')          $n - Dataset ID (e.g. 459)  e.g. /$i/$a                                                              |
+
+[Project]: ../CodeTutorials/Glossary.md#Project
+[ExtractionProgress]: ../CodeTutorials/Glossary.md#ExtractionProgress
+[Catalogue]: ../CodeTutorials/Glossary.md#Catalogue
+
+
+## Notes on updating an extraction
+Using the "Append Data If Table Exists" option within this extraction destination component will allow you to re-extract additional information to a database, however there a number of caveats and gotchas with this.
+* Archive Table
+    * Similar to data loads, these database tables will come with an _Archive table that will auto-populate when new extractions are ran against the database.
+* Data Structure changing
+    * While the extractor can handle columns being removed, it does not support columns being added beyond the first extraction. For this you will need to write the data to a new database table. It is recommended that the extraction is cloned in this case.