From b16cb8c786800131a3bb5b02e58b3d1328b28673 Mon Sep 17 00:00:00 2001 From: James A Sutherland <> Date: Fri, 3 Nov 2023 17:50:48 -0500 Subject: [PATCH 1/5] Implement first pass at platform export tool --- .../ExecuteCommandExportDatabaseToDir.cs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs new file mode 100644 index 0000000000..0db66d7a34 --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs @@ -0,0 +1,37 @@ +// Copyright (c) The University of Dundee 2018-2023 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using System.IO; +using System.Linq; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Repositories; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +// Dump all compatible objects from the current platform database(s) into a +// YAML/JSON directory for SQL-free operation. +public class ExecuteCommandExportDatabaseToDir : BasicCommandExecution +{ + private readonly IBasicActivateItems _activator; + private readonly DirectoryInfo _target; + + public ExecuteCommandExportDatabaseToDir(IBasicActivateItems activator, [DemandsInitialization("Where the platform directory should be created")] string target) + { + _target = new DirectoryInfo(target); + _activator = activator; + } + + public override void Execute() + { + base.Execute(); + var repo = new YamlRepository(_target); + var types = repo.GetCompatibleTypes(); + foreach (var o in repo.GetCompatibleTypes().SelectMany(t=>_activator.GetRepositoryFor(t).GetAllObjects(t))) + { + repo.SaveToDatabase(o); + } + } +} \ No newline at end of file From 5b5b66f8db223333d9af26d0e8f6722135cf7c99 Mon Sep 17 00:00:00 2001 From: James A Sutherland <> Date: Mon, 6 Nov 2023 09:27:11 -0600 Subject: [PATCH 2/5] Remove dead code, add progress report --- .../AtomicCommands/ExecuteCommandExportDatabaseToDir.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs index 0db66d7a34..755d4fdb30 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs @@ -28,10 +28,11 @@ public override void Execute() { base.Execute(); var repo = new YamlRepository(_target); - var types = repo.GetCompatibleTypes(); - foreach (var o in repo.GetCompatibleTypes().SelectMany(t=>_activator.GetRepositoryFor(t).GetAllObjects(t))) + foreach (var t in repo.GetCompatibleTypes()) { - repo.SaveToDatabase(o); + Console.WriteLine(t.FullName); + foreach (var o in t.SelectMany(t=>_activator.GetRepositoryFor(t).GetAllObjects(t))) + repo.SaveToDatabase(o); } } } \ No newline at end of file From 0fee40236c78f4a5dfb8a1a12ffcf9fe37d5a0b0 Mon Sep 17 00:00:00 2001 From: James A Sutherland <> Date: Mon, 6 Nov 2023 09:39:33 -0600 Subject: [PATCH 3/5] Syntax fix --- .../AtomicCommands/ExecuteCommandExportDatabaseToDir.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs index 755d4fdb30..67edc1fa14 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs @@ -4,6 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using System; using System.IO; using System.Linq; using Rdmp.Core.Curation.Data; @@ -31,7 +32,7 @@ public override void Execute() foreach (var t in repo.GetCompatibleTypes()) { Console.WriteLine(t.FullName); - foreach (var o in t.SelectMany(t=>_activator.GetRepositoryFor(t).GetAllObjects(t))) + foreach (var o in _activator.GetRepositoryFor(t).GetAllObjects(t)) repo.SaveToDatabase(o); } } From 039cdcc5213798d30ea8235c570cb729ad858b70 Mon Sep 17 00:00:00 2001 From: James A Sutherland <> Date: Mon, 6 Nov 2023 09:40:00 -0600 Subject: [PATCH 4/5] Note command addition in CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 425572477a..97f24ce56c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Improved file transfer (FTP/SFTP/FTPS) support +- Add command to dump current platform DB to directory ## [8.1.0] - 2023-09-19 From 902d8248ad194f99ad68a8e719afe52080394fca Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 27 Nov 2023 08:27:00 +0000 Subject: [PATCH 5/5] ignore certain dqe types --- .../ExecuteCommandExportDatabaseToDir.cs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs index 67edc1fa14..eefdba5401 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandExportDatabaseToDir.cs @@ -5,8 +5,10 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Linq.Expressions; using Rdmp.Core.Curation.Data; using Rdmp.Core.Repositories; @@ -25,15 +27,26 @@ public ExecuteCommandExportDatabaseToDir(IBasicActivateItems activator, [Demands _activator = activator; } + private readonly List ignoreList = new() { "Rdmp.Core.DataQualityEngine.Data.DQEGraphAnnotation", "Rdmp.Core.DataQualityEngine.Data.Evaluation" }; + public override void Execute() { base.Execute(); var repo = new YamlRepository(_target); foreach (var t in repo.GetCompatibleTypes()) { - Console.WriteLine(t.FullName); - foreach (var o in _activator.GetRepositoryFor(t).GetAllObjects(t)) - repo.SaveToDatabase(o); + if (ignoreList.Contains(t.FullName)) continue; + try + { + Console.WriteLine(t.FullName); + foreach (var o in _activator.GetRepositoryFor(t).GetAllObjects(t)) + repo.SaveToDatabase(o); + } + catch(Exception) + { + Console.WriteLine($"Unable to find repo for {t.FullName}"); + } + } } } \ No newline at end of file