From dfa3ca6c1d2c5dd772450c768bee92c458ca692d Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 4 Aug 2023 15:42:29 +0100 Subject: [PATCH 01/14] make external serverremovable --- Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs | 1 + .../CatalogueDatabase/up/077_DropExternalServerFK.sql | 5 +++++ Rdmp.Core/Rdmp.Core.csproj | 1 + 3 files changed, 7 insertions(+) create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql diff --git a/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs b/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs index 19d486edfb..896cc02736 100644 --- a/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs +++ b/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs @@ -46,6 +46,7 @@ public string Name set => SetField(ref _name, value); } + /// /// If the database was created by an RDMP schema (or plugin schema) this will contain the name of the dll which holds the schema e.g. DataQualityEngine.Database and was /// responsible for creating the database. This determines what roles RDMP lets the database play. diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql new file mode 100644 index 0000000000..a869683657 --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql @@ -0,0 +1,5 @@ +--Version:7.0.0 +--Description: Updates database to support custom folders on CohortIdentificationConfiguration and LoadMetadata + +-- Add to LoadMetadata table + alter table Catalogue drop constraint FK_Catalogue_ExternalDatabaseServer diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 51048c4e84..a054958592 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -231,6 +231,7 @@ + From 68042038f04210901b3d5ffa0962e57f08ce01c0 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 4 Aug 2023 16:05:10 +0100 Subject: [PATCH 02/14] no longer throwing out errors --- Rdmp.Core/Curation/Data/Catalogue.cs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 0b843e1900..0818d7a13a 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -521,10 +521,17 @@ public LoadMetadata LoadMetadata /// [NoMappingToDatabase] - public ExternalDatabaseServer LiveLoggingServer => - LiveLoggingServer_ID == null - ? null - : Repository.GetObjectByID((int)LiveLoggingServer_ID); + public ExternalDatabaseServer LiveLoggingServer { + get{ + if(LiveLoggingServer_ID != null){ + ExternalDatabaseServer db = Repository.GetObjectByID((int)LiveLoggingServer_ID); + if(db != null){ + return db; + } + } + return null; + } + } /// [NoMappingToDatabase] From 541101783b231a24e9bab26f0a69bcd29e39e49b Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 4 Aug 2023 16:23:31 +0100 Subject: [PATCH 03/14] add catalogue fix --- Rdmp.Core/Curation/Data/Catalogue.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 0818d7a13a..114e5b123d 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -523,11 +523,12 @@ public LoadMetadata LoadMetadata [NoMappingToDatabase] public ExternalDatabaseServer LiveLoggingServer { get{ - if(LiveLoggingServer_ID != null){ - ExternalDatabaseServer db = Repository.GetObjectByID((int)LiveLoggingServer_ID); - if(db != null){ - return db; - } + if(LiveLoggingServer_ID != null){ + // ExternalDatabaseServer[] dbs = Repository.GetAllObjects(); + ExternalDatabaseServer[] dbs = Repository.GetAllObjectsWhere("id",(int)LiveLoggingServer_ID); + if(dbs.Length > 0){ + return dbs.First(); + } } return null; } From 83502721ba88c13ac4bd24a3a13ce55bd2a07d35 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 4 Aug 2023 16:45:00 +0100 Subject: [PATCH 04/14] added todo reminder --- .../Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql index a869683657..758ad01e34 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql @@ -3,3 +3,5 @@ -- Add to LoadMetadata table alter table Catalogue drop constraint FK_Catalogue_ExternalDatabaseServer +-- this might be better by making specific serveres ignorable rather than deleting the FK +-- would have to check if a logging server is always generated on launch \ No newline at end of file From 1646503c538a8b2ac11d849773e62ad876297683 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 7 Aug 2023 09:45:03 +0100 Subject: [PATCH 05/14] working startup option --- .../ExampleDatasetsCreationTests.cs | 2 +- .../Engine/Integration/ExcelDatabaseTests.cs | 2 +- ...CataloguePipelinesAndReferencesCreation.cs | 44 ++++++++++--------- .../PlatformDatabaseCreation.cs | 2 +- .../PlatformDatabaseCreationOptions.cs | 4 ++ .../ChoosePlatformDatabasesUI.Designer.cs | 15 ++++++- .../ChoosePlatformDatabasesUI.cs | 2 +- 7 files changed, 45 insertions(+), 26 deletions(-) diff --git a/Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs b/Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs index c2927014d1..7080cbce21 100644 --- a/Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs +++ b/Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs @@ -29,7 +29,7 @@ public void Test_ExampleDatasetsCreation() //create the pipelines var pipes = new CataloguePipelinesAndReferencesCreation(RepositoryLocator,null,null); - pipes.CreatePipelines(); + pipes.CreatePipelines(new PlatformDatabaseCreationOptions {}); //create all the stuff var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer); diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/ExcelDatabaseTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/ExcelDatabaseTests.cs index c509233633..9be31aba43 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/ExcelDatabaseTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/ExcelDatabaseTests.cs @@ -28,7 +28,7 @@ public void TestLoadingFileWithTrailingDotsInHeader() // Create the 'out of the box' RDMP pipelines (which includes an excel bulk importer pipeline) var creator = new CataloguePipelinesAndReferencesCreation( RepositoryLocator, UnitTestLoggingConnectionString, DataQualityEngineConnectionString); - creator.CreatePipelines(); + creator.CreatePipelines(new PlatformDatabaseCreationOptions {}); // find the excel loading pipeline var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID).FirstOrDefault(p => p.Name.Contains("BULK INSERT: Excel File")); diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs index 1ce1a8d72d..ca781dc915 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs @@ -48,26 +48,27 @@ private void DoStartup() var startup = new Startup.Startup(new EnvironmentInfo(),_repositoryLocator); startup.DoStartup(new IgnoreAllErrorsCheckNotifier()); } - private void CreateServers() + private void CreateServers(PlatformDatabaseCreationOptions options) { var defaults = _repositoryLocator.CatalogueRepository; - - _edsLogging = new ExternalDatabaseServer(_repositoryLocator.CatalogueRepository, "Logging",new LoggingDatabasePatcher()) + if(!options.SkipLoggingServer){ + _edsLogging = new ExternalDatabaseServer(_repositoryLocator.CatalogueRepository, "Logging",new LoggingDatabasePatcher()) + { + Server = _logging.DataSource, + Database = _logging.InitialCatalog + }; + + if(_logging.UserID != null) { - Server = _logging.DataSource, - Database = _logging.InitialCatalog - }; + _edsLogging.Username = _logging.UserID; + _edsLogging.Password = _logging.Password; + } - if(_logging.UserID != null) - { - _edsLogging.Username = _logging.UserID; - _edsLogging.Password = _logging.Password; + _edsLogging.SaveToDatabase(); + defaults.SetDefault(PermissableDefaults.LiveLoggingServer_ID, _edsLogging); + Console.WriteLine("Successfully configured default logging server"); } - _edsLogging.SaveToDatabase(); - defaults.SetDefault(PermissableDefaults.LiveLoggingServer_ID, _edsLogging); - Console.WriteLine("Successfully configured default logging server"); - var edsDQE = new ExternalDatabaseServer(_repositoryLocator.CatalogueRepository, "DQE", new DataQualityEnginePatcher()) { Server = _dqe.DataSource, @@ -104,7 +105,7 @@ private void CreateServers() Console.WriteLine("Successfully configured RAW server"); } - public void CreatePipelines() + public void CreatePipelines(PlatformDatabaseCreationOptions options) { var bulkInsertCsvPipe = CreatePipeline("BULK INSERT: CSV Import File (manual column-type editing)", typeof(DelimitedFlatFileDataFlowSource), typeof(DataTableUploadDestination)); @@ -117,9 +118,10 @@ public void CreatePipelines() SetComponentProperties(bulkInsertCsvPipewithAdjuster.Source, "Separator", ","); SetComponentProperties(bulkInsertCsvPipewithAdjuster.Source, "StronglyTypeInput", false); - SetComponentProperties(bulkInsertCsvPipe.Destination, "LoggingServer", _edsLogging); - SetComponentProperties(bulkInsertCsvPipewithAdjuster.Destination, "LoggingServer", _edsLogging); - + if(!options.SkipLoggingServer){ + SetComponentProperties(bulkInsertCsvPipe.Destination, "LoggingServer", _edsLogging); + SetComponentProperties(bulkInsertCsvPipewithAdjuster.Destination, "LoggingServer", _edsLogging); + } var createCohortFromCSV = CreatePipeline("CREATE COHORT:From CSV File",typeof (DelimitedFlatFileDataFlowSource), typeof (BasicCohortDestination)); SetComponentProperties(createCohortFromCSV.Source, "Separator", ","); @@ -184,11 +186,11 @@ private Pipeline CreatePipeline(string nameOfPipe, Type sourceType, Type destina return pipe; } - public void Create() + public void Create(PlatformDatabaseCreationOptions options) { DoStartup(); - CreateServers(); - CreatePipelines(); + CreateServers(options); + CreatePipelines(options); } } \ No newline at end of file diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs index da0719df2a..97f8216e37 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs @@ -47,7 +47,7 @@ public static void CreatePlatformDatabases(PlatformDatabaseCreationOptions optio if (!options.SkipPipelines) { var creator = new CataloguePipelinesAndReferencesCreation(repo, logging, dqe); - creator.Create(); + creator.Create(options);//todo need to figure out how/where to set the options } if(options.ExampleDatasets || options.Nightmare) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs index 6e337424c7..6714777767 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs @@ -37,6 +37,9 @@ public class PlatformDatabaseCreationOptions [Option('k', "Skip Pipelines", Default = false, HelpText = "Skips creating the default Pipelines and Managed Server References in the Catalogue database once created.")] public bool SkipPipelines { get; set; } + + [Option('l', "Skip Logging Server", Default = false, HelpText = "Skips creating the default logging server in the Catalogue database once created. Is superseeded by 'Skip Pipelines'")] + public bool SkipLoggingServer { get; set; } [Option('e', "ExampleDatasets", Default = false, HelpText = "Create example datasets, projects, extraction configurations and cohort queries")] public bool ExampleDatasets{get;set;} @@ -65,6 +68,7 @@ public class PlatformDatabaseCreationOptions [Option(Required = false, HelpText = "Optional connection string keywords to use e.g. \"Key1=Value1; Key2=Value2\". When using this option you must manually specify IntegratedSecurity if required.")] public string OtherKeywords { get; set; } + [Usage] public static IEnumerable Examples { diff --git a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs index 81db0fbc68..4103527b64 100644 --- a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs +++ b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs @@ -70,6 +70,7 @@ private void InitializeComponent() this.label11 = new System.Windows.Forms.Label(); this.tbRowCount = new System.Windows.Forms.TextBox(); this.cbCreateExampleDatasets = new System.Windows.Forms.CheckBox(); + this.cbSkipLoggingServer = new System.Windows.Forms.CheckBox(); this.tbSeed = new System.Windows.Forms.TextBox(); this.tbOtherKeywords = new System.Windows.Forms.TextBox(); this.tbCreateDatabaseTimeout = new System.Windows.Forms.TextBox(); @@ -375,6 +376,7 @@ private void InitializeComponent() // gbCreateNew // this.gbCreateNew.Controls.Add(this.cbCreateExampleDatasets); + this.gbCreateNew.Controls.Add(this.cbSkipLoggingServer); this.gbCreateNew.Controls.Add(this.gbSqlAuthentication); this.gbCreateNew.Controls.Add(this.gbExampleDatasets); this.gbCreateNew.Controls.Add(this.tbOtherKeywords); @@ -522,7 +524,17 @@ private void InitializeComponent() this.cbCreateExampleDatasets.Text = "Example Datasets"; this.cbCreateExampleDatasets.UseVisualStyleBackColor = true; this.cbCreateExampleDatasets.CheckedChanged += new System.EventHandler(this.cbCreateExampleDatasets_CheckedChanged); - // + // + // cbSkipLoggingServer + // + this.cbSkipLoggingServer.AutoSize = true; + this.cbSkipLoggingServer.Location = new System.Drawing.Point(543, 156); + this.cbSkipLoggingServer.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); + this.cbSkipLoggingServer.Name = "cbSkipLoggingServer"; + this.cbSkipLoggingServer.Size = new System.Drawing.Size(118, 19); + this.cbSkipLoggingServer.TabIndex = 9; + this.cbSkipLoggingServer.Text = "Do not create a logging server"; + this.cbSkipLoggingServer.UseVisualStyleBackColor = true; // tbSeed // this.tbSeed.Location = new System.Drawing.Point(94, 22); @@ -672,6 +684,7 @@ private void InitializeComponent() private System.Windows.Forms.Label label4; private System.Windows.Forms.Label label9; private System.Windows.Forms.CheckBox cbCreateExampleDatasets; + private System.Windows.Forms.CheckBox cbSkipLoggingServer; private System.Windows.Forms.TextBox tbSeed; private System.Windows.Forms.Label label10; private System.Windows.Forms.TextBox tbRowCount; diff --git a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs index aea6f5dac7..b8a3e14a76 100644 --- a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs +++ b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs @@ -267,6 +267,7 @@ private void btnCreateSuite_Click(object sender, EventArgs e) Username = tbUsername.Text, Password = tbPassword.Text, ExampleDatasets = cbCreateExampleDatasets.Checked, + SkipLoggingServer = cbSkipLoggingServer.Checked, Seed = _seed, NumberOfPeople = _peopleCount, NumberOfRowsPerDataset = _rowCount, @@ -447,5 +448,4 @@ private void cbCreateExampleDatasets_CheckedChanged(object sender, EventArgs e) { gbExampleDatasets.Enabled = cbCreateExampleDatasets.Checked; } - } \ No newline at end of file From 76c5cc870be5f88f63c5d46c5e1e228b1098dc22 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 7 Aug 2023 09:53:39 +0100 Subject: [PATCH 06/14] remove fk drop --- .../CatalogueDatabase/up/077_DropExternalServerFK.sql | 7 ------- Rdmp.Core/Rdmp.Core.csproj | 1 - 2 files changed, 8 deletions(-) delete mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql deleted file mode 100644 index 758ad01e34..0000000000 --- a/Rdmp.Core/Databases/CatalogueDatabase/up/077_DropExternalServerFK.sql +++ /dev/null @@ -1,7 +0,0 @@ ---Version:7.0.0 ---Description: Updates database to support custom folders on CohortIdentificationConfiguration and LoadMetadata - --- Add to LoadMetadata table - alter table Catalogue drop constraint FK_Catalogue_ExternalDatabaseServer --- this might be better by making specific serveres ignorable rather than deleting the FK --- would have to check if a logging server is always generated on launch \ No newline at end of file diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index a054958592..51048c4e84 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -231,7 +231,6 @@ - From 560c68d783db87b5c00f4ad3df7f81c150c1b09a Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 7 Aug 2023 10:35:55 +0100 Subject: [PATCH 07/14] fix creation issue --- .../CataloguePipelinesAndReferencesCreation.cs | 2 +- .../DatabaseCreation/PlatformDatabaseCreation.cs | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs index ca781dc915..88569f8cc7 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs @@ -91,7 +91,7 @@ private void CreateServers(PlatformDatabaseCreationOptions options) }; //We are expecting a single username/password for everything here, so just use the dqe one - if (_dqe.UserID != null) + if (_dqe.UserID != null && _logging != null) { if (_logging.UserID != _dqe.UserID || _logging.Password != _dqe.Password) throw new Exception("DQE uses sql authentication but the credentials are not the same as the logging db. Could not pick a single set of credentials to use for the RAW server entry"); diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs index 97f8216e37..8fbbea2658 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs @@ -38,8 +38,11 @@ public static void CreatePlatformDatabases(PlatformDatabaseCreationOptions optio Create(DefaultDataExportDatabaseName, new DataExportPatcher(), options); var dqe = Create(DefaultDQEDatabaseName, new DataQualityEnginePatcher(), options); - var logging = Create(DefaultLoggingDatabaseName, new LoggingDatabasePatcher(), options); - + + SqlConnectionStringBuilder logging = null; + if(!options.SkipLoggingServer){ + logging = Create(DefaultLoggingDatabaseName, new LoggingDatabasePatcher(), options); + } CatalogueRepository.SuppressHelpLoading = true; var repo = new PlatformDatabaseCreationRepositoryFinder(options); @@ -47,7 +50,7 @@ public static void CreatePlatformDatabases(PlatformDatabaseCreationOptions optio if (!options.SkipPipelines) { var creator = new CataloguePipelinesAndReferencesCreation(repo, logging, dqe); - creator.Create(options);//todo need to figure out how/where to set the options + creator.Create(options); } if(options.ExampleDatasets || options.Nightmare) From 0ec0c43e57b1794c4fa9b72014054f03120ae745 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 7 Aug 2023 13:32:15 +0100 Subject: [PATCH 08/14] add todo --- Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs b/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs index 896cc02736..e1578f1582 100644 --- a/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs +++ b/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs @@ -289,6 +289,7 @@ public bool DiscoverExistence(DataAccessContext context,out string reason) public override void DeleteInDatabase() { + //here if its a logging serber, remove all FK referneces first base.DeleteInDatabase(); // normally in database schema deleting an ExternalDatabaseServer will cascade to clear defaults From 91e76384ce7cf0e9f6b2f210b7030df0ae0a74cc Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 7 Aug 2023 15:37:11 +0100 Subject: [PATCH 09/14] working delete --- Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs b/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs index e1578f1582..73dbc141fa 100644 --- a/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs +++ b/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs @@ -289,7 +289,15 @@ public bool DiscoverExistence(DataAccessContext context,out string reason) public override void DeleteInDatabase() { - //here if its a logging serber, remove all FK referneces first + + if (WasCreatedBy(new LoggingDatabasePatcher())){ + //If you're trying to delete a logging server, remove all references to it first + Catalogue[] catalogues = Repository.GetAllObjectsWhere("LiveLoggingServer_ID",base.ID); + foreach(Catalogue catalogue in catalogues){ + catalogue.LiveLoggingServer_ID = null; + catalogue.SaveToDatabase(); + } + } base.DeleteInDatabase(); // normally in database schema deleting an ExternalDatabaseServer will cascade to clear defaults From 3f5d7440aafe106c5fb4241c41692f14b34219eb Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 7 Aug 2023 16:10:13 +0100 Subject: [PATCH 10/14] invert checkbox ui --- .../CataloguePipelinesAndReferencesCreation.cs | 3 ++- Rdmp.Core/Curation/Data/Catalogue.cs | 1 - Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs | 1 - Rdmp.Core/GlobalStrings.Designer.cs | 6 ++++++ Rdmp.Core/GlobalStrings.resx | 3 +++ Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs | 4 +++- Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs | 2 +- 7 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs index 88569f8cc7..e8d6d84459 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs @@ -91,7 +91,8 @@ private void CreateServers(PlatformDatabaseCreationOptions options) }; //We are expecting a single username/password for everything here, so just use the dqe one - if (_dqe.UserID != null && _logging != null) + bool hasLoggingDB = _logging != null && _logging.UserID != null; + if (_dqe.UserID != null && hasLoggingDB) { if (_logging.UserID != _dqe.UserID || _logging.Password != _dqe.Password) throw new Exception("DQE uses sql authentication but the credentials are not the same as the logging db. Could not pick a single set of credentials to use for the RAW server entry"); diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 114e5b123d..b510613d1f 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -524,7 +524,6 @@ public LoadMetadata LoadMetadata public ExternalDatabaseServer LiveLoggingServer { get{ if(LiveLoggingServer_ID != null){ - // ExternalDatabaseServer[] dbs = Repository.GetAllObjects(); ExternalDatabaseServer[] dbs = Repository.GetAllObjectsWhere("id",(int)LiveLoggingServer_ID); if(dbs.Length > 0){ return dbs.First(); diff --git a/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs b/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs index 73dbc141fa..08b9e2d145 100644 --- a/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs +++ b/Rdmp.Core/Curation/Data/ExternalDatabaseServer.cs @@ -46,7 +46,6 @@ public string Name set => SetField(ref _name, value); } - /// /// If the database was created by an RDMP schema (or plugin schema) this will contain the name of the dll which holds the schema e.g. DataQualityEngine.Database and was /// responsible for creating the database. This determines what roles RDMP lets the database play. diff --git a/Rdmp.Core/GlobalStrings.Designer.cs b/Rdmp.Core/GlobalStrings.Designer.cs index 61f0546536..cb2d918533 100644 --- a/Rdmp.Core/GlobalStrings.Designer.cs +++ b/Rdmp.Core/GlobalStrings.Designer.cs @@ -230,5 +230,11 @@ public static string TriggerStatusIsCurrently { return ResourceManager.GetString("TriggerStatusIsCurrently", resourceCulture); } } + + public static string CreateALoggingServer { + get { + return ResourceManager.GetString("CreateALoggingServer"); + } + } } } diff --git a/Rdmp.Core/GlobalStrings.resx b/Rdmp.Core/GlobalStrings.resx index 52bbed2f06..35bad529ce 100644 --- a/Rdmp.Core/GlobalStrings.resx +++ b/Rdmp.Core/GlobalStrings.resx @@ -174,4 +174,7 @@ Catalogue from File... + + Create a logging server + \ No newline at end of file diff --git a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs index 4103527b64..95b84807a2 100644 --- a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs +++ b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs @@ -1,4 +1,5 @@ using System; +using Rdmp.Core; using Rdmp.UI.SimpleControls; @@ -533,7 +534,8 @@ private void InitializeComponent() this.cbSkipLoggingServer.Name = "cbSkipLoggingServer"; this.cbSkipLoggingServer.Size = new System.Drawing.Size(118, 19); this.cbSkipLoggingServer.TabIndex = 9; - this.cbSkipLoggingServer.Text = "Do not create a logging server"; + this.cbSkipLoggingServer.Checked = true; + this.cbSkipLoggingServer.Text = Core.GlobalStrings.CreateALoggingServer; this.cbSkipLoggingServer.UseVisualStyleBackColor = true; // tbSeed // diff --git a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs index b8a3e14a76..5a94f44860 100644 --- a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs +++ b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs @@ -267,7 +267,7 @@ private void btnCreateSuite_Click(object sender, EventArgs e) Username = tbUsername.Text, Password = tbPassword.Text, ExampleDatasets = cbCreateExampleDatasets.Checked, - SkipLoggingServer = cbSkipLoggingServer.Checked, + SkipLoggingServer = !cbSkipLoggingServer.Checked, Seed = _seed, NumberOfPeople = _peopleCount, NumberOfRowsPerDataset = _rowCount, From 8dfc62ed1c6a8f7e6dc9e4dc1d613ce8a4494ae4 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 8 Aug 2023 14:56:27 +0100 Subject: [PATCH 11/14] add throw for unexpected no logging server --- .../DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs index e8d6d84459..1407840224 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs @@ -54,7 +54,7 @@ private void CreateServers(PlatformDatabaseCreationOptions options) if(!options.SkipLoggingServer){ _edsLogging = new ExternalDatabaseServer(_repositoryLocator.CatalogueRepository, "Logging",new LoggingDatabasePatcher()) { - Server = _logging.DataSource, + Server = _logging?.DataSource ?? throw new InvalidOperationException("Null logging database provided"), Database = _logging.InitialCatalog }; From c22c2bfee7f3f201b621cb9db543a508768d3247 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 9 Aug 2023 08:29:29 +0100 Subject: [PATCH 12/14] rename variables to match descriptions --- ...CataloguePipelinesAndReferencesCreation.cs | 4 +-- .../PlatformDatabaseCreation.cs | 2 +- .../PlatformDatabaseCreationOptions.cs | 4 +-- .../ChoosePlatformDatabasesUI.Designer.cs | 28 +++++++++---------- .../ChoosePlatformDatabasesUI.cs | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs index f4e6915393..54cfbef7f9 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs @@ -52,7 +52,7 @@ private void DoStartup() private void CreateServers(PlatformDatabaseCreationOptions options) { var defaults = _repositoryLocator.CatalogueRepository; - if(!options.SkipLoggingServer){ + if(!options.CreateLoggingServer){ _edsLogging = new ExternalDatabaseServer(_repositoryLocator.CatalogueRepository, "Logging",new LoggingDatabasePatcher()) { Server = _logging?.DataSource ?? throw new InvalidOperationException("Null logging database provided"), @@ -123,7 +123,7 @@ public void CreatePipelines(PlatformDatabaseCreationOptions options) SetComponentProperties(bulkInsertCsvPipewithAdjuster.Source, "Separator", ","); SetComponentProperties(bulkInsertCsvPipewithAdjuster.Source, "StronglyTypeInput", false); - if(!options.SkipLoggingServer){ + if(!options.CreateLoggingServer){ SetComponentProperties(bulkInsertCsvPipe.Destination, "LoggingServer", _edsLogging); SetComponentProperties(bulkInsertCsvPipewithAdjuster.Destination, "LoggingServer", _edsLogging); } diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs index b6501e27a6..17c9b4d845 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs @@ -40,7 +40,7 @@ public static void CreatePlatformDatabases(PlatformDatabaseCreationOptions optio var dqe = Create(DefaultDQEDatabaseName, new DataQualityEnginePatcher(), options); SqlConnectionStringBuilder logging = null; - if(!options.SkipLoggingServer){ + if(options.CreateLoggingServer){ logging = Create(DefaultLoggingDatabaseName, new LoggingDatabasePatcher(), options); } CatalogueRepository.SuppressHelpLoading = true; diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs index a5f8b116a9..401eda1ce2 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs @@ -43,8 +43,8 @@ public class PlatformDatabaseCreationOptions "Skips creating the default Pipelines and Managed Server References in the Catalogue database once created.")] public bool SkipPipelines { get; set; } - [Option('l', "Skip Logging Server", Default = false, HelpText = "Skips creating the default logging server in the Catalogue database once created. Is superseeded by 'Skip Pipelines'")] - public bool SkipLoggingServer { get; set; } + [Option('l', "Create Logging Server", Default = true, HelpText = "Create the default logging server in the Catalogue database once created. Is superseeded by 'Skip Pipelines'")] + public bool CreateLoggingServer { get; set; } [Option('e', "ExampleDatasets", Default = false, HelpText = "Create example datasets, projects, extraction configurations and cohort queries")] diff --git a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs index 95b84807a2..fe75ce878d 100644 --- a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs +++ b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.Designer.cs @@ -71,7 +71,7 @@ private void InitializeComponent() this.label11 = new System.Windows.Forms.Label(); this.tbRowCount = new System.Windows.Forms.TextBox(); this.cbCreateExampleDatasets = new System.Windows.Forms.CheckBox(); - this.cbSkipLoggingServer = new System.Windows.Forms.CheckBox(); + this.cbCreateLoggingServer = new System.Windows.Forms.CheckBox(); this.tbSeed = new System.Windows.Forms.TextBox(); this.tbOtherKeywords = new System.Windows.Forms.TextBox(); this.tbCreateDatabaseTimeout = new System.Windows.Forms.TextBox(); @@ -377,7 +377,7 @@ private void InitializeComponent() // gbCreateNew // this.gbCreateNew.Controls.Add(this.cbCreateExampleDatasets); - this.gbCreateNew.Controls.Add(this.cbSkipLoggingServer); + this.gbCreateNew.Controls.Add(this.cbCreateLoggingServer); this.gbCreateNew.Controls.Add(this.gbSqlAuthentication); this.gbCreateNew.Controls.Add(this.gbExampleDatasets); this.gbCreateNew.Controls.Add(this.tbOtherKeywords); @@ -526,17 +526,17 @@ private void InitializeComponent() this.cbCreateExampleDatasets.UseVisualStyleBackColor = true; this.cbCreateExampleDatasets.CheckedChanged += new System.EventHandler(this.cbCreateExampleDatasets_CheckedChanged); // - // cbSkipLoggingServer - // - this.cbSkipLoggingServer.AutoSize = true; - this.cbSkipLoggingServer.Location = new System.Drawing.Point(543, 156); - this.cbSkipLoggingServer.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); - this.cbSkipLoggingServer.Name = "cbSkipLoggingServer"; - this.cbSkipLoggingServer.Size = new System.Drawing.Size(118, 19); - this.cbSkipLoggingServer.TabIndex = 9; - this.cbSkipLoggingServer.Checked = true; - this.cbSkipLoggingServer.Text = Core.GlobalStrings.CreateALoggingServer; - this.cbSkipLoggingServer.UseVisualStyleBackColor = true; + // cbCreateLoggingServer + // + this.cbCreateLoggingServer.AutoSize = true; + this.cbCreateLoggingServer.Location = new System.Drawing.Point(543, 156); + this.cbCreateLoggingServer.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); + this.cbCreateLoggingServer.Name = "cbCreateLoggingServer"; + this.cbCreateLoggingServer.Size = new System.Drawing.Size(118, 19); + this.cbCreateLoggingServer.TabIndex = 9; + this.cbCreateLoggingServer.Checked = true; + this.cbCreateLoggingServer.Text = Core.GlobalStrings.CreateALoggingServer; + this.cbCreateLoggingServer.UseVisualStyleBackColor = true; // tbSeed // this.tbSeed.Location = new System.Drawing.Point(94, 22); @@ -686,7 +686,7 @@ private void InitializeComponent() private System.Windows.Forms.Label label4; private System.Windows.Forms.Label label9; private System.Windows.Forms.CheckBox cbCreateExampleDatasets; - private System.Windows.Forms.CheckBox cbSkipLoggingServer; + private System.Windows.Forms.CheckBox cbCreateLoggingServer; private System.Windows.Forms.TextBox tbSeed; private System.Windows.Forms.Label label10; private System.Windows.Forms.TextBox tbRowCount; diff --git a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs index 28ead15af4..c81b612611 100644 --- a/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs +++ b/Rdmp.UI/LocationsMenu/ChoosePlatformDatabasesUI.cs @@ -271,7 +271,7 @@ private void btnCreateSuite_Click(object sender, EventArgs e) Username = tbUsername.Text, Password = tbPassword.Text, ExampleDatasets = cbCreateExampleDatasets.Checked, - SkipLoggingServer = !cbSkipLoggingServer.Checked, + CreateLoggingServer = cbCreateLoggingServer.Checked, Seed = _seed, NumberOfPeople = _peopleCount, NumberOfRowsPerDataset = _rowCount, From 83fa74e35fde67711f997b9aa0f919766a842ce2 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 9 Aug 2023 08:30:58 +0100 Subject: [PATCH 13/14] fix bad merge --- Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs b/Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs index fe3cfa701e..5429060c0a 100644 --- a/Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs +++ b/Rdmp.Core.Tests/CommandLine/ExampleDatasetsCreationTests.cs @@ -28,13 +28,8 @@ public void Test_ExampleDatasetsCreation() Assert.AreEqual(0, CatalogueRepository.GetAllObjects().Length); //create the pipelines -<<<<<<< HEAD var pipes = new CataloguePipelinesAndReferencesCreation(RepositoryLocator,null,null); pipes.CreatePipelines(new PlatformDatabaseCreationOptions {}); -======= - var pipes = new CataloguePipelinesAndReferencesCreation(RepositoryLocator, null, null); - pipes.CreatePipelines(); ->>>>>>> a9b3318811dda8ffea546bf4a0e5a0e0278d4a00 //create all the stuff var db = GetCleanedServer(FAnsi.DatabaseType.MicrosoftSQLServer); From 5f7ffa2b3a9c8672f149bd88853ac2bd3b87915a Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 9 Aug 2023 08:32:45 +0100 Subject: [PATCH 14/14] fix autosave issue --- .../CataloguePipelinesAndReferencesCreation.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs index 54cfbef7f9..bad1a9bd27 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/CataloguePipelinesAndReferencesCreation.cs @@ -52,7 +52,7 @@ private void DoStartup() private void CreateServers(PlatformDatabaseCreationOptions options) { var defaults = _repositoryLocator.CatalogueRepository; - if(!options.CreateLoggingServer){ + if(options.CreateLoggingServer){ _edsLogging = new ExternalDatabaseServer(_repositoryLocator.CatalogueRepository, "Logging",new LoggingDatabasePatcher()) { Server = _logging?.DataSource ?? throw new InvalidOperationException("Null logging database provided"), @@ -123,7 +123,7 @@ public void CreatePipelines(PlatformDatabaseCreationOptions options) SetComponentProperties(bulkInsertCsvPipewithAdjuster.Source, "Separator", ","); SetComponentProperties(bulkInsertCsvPipewithAdjuster.Source, "StronglyTypeInput", false); - if(!options.CreateLoggingServer){ + if(options.CreateLoggingServer){ SetComponentProperties(bulkInsertCsvPipe.Destination, "LoggingServer", _edsLogging); SetComponentProperties(bulkInsertCsvPipewithAdjuster.Destination, "LoggingServer", _edsLogging); }