Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RDMP-15 Use .bak files as Data Loads #1656

Merged
merged 9 commits into from
Oct 26, 2023
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Rdmp.Core/CommandExecution/AtomicCommandFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -545,6 +545,8 @@ public IEnumerable<IAtomicCommand> CreateCommands(object o)
lsn.LoadMetadata, lsn.LoadStage);
yield return new ExecuteCommandCreateNewFileBasedProcessTask(_activator, ProcessTaskType.Executable,
lsn.LoadMetadata, lsn.LoadStage);
yield return new ExecuteCommandCreateNewFileBasedProcessTask(_activator, ProcessTaskType.SQLBakFile,
lsn.LoadMetadata, lsn.LoadStage);
}

if (Is(o, out LoadDirectoryNode ldn))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

using System;
using System.IO;
using System.Linq;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.Icons.IconOverlays;
Expand Down Expand Up @@ -41,8 +42,9 @@ public ExecuteCommandCreateNewFileBasedProcessTask(IBasicActivateItems activator
SetImpossible("Could not construct LoadDirectory");
}

if (taskType is not (ProcessTaskType.SQLFile or ProcessTaskType.Executable))
SetImpossible("Only SQLFile and Executable task types are supported by this command");
ProcessTaskType[] AcceptedProcessTaskTypes= { ProcessTaskType.SQLFile, ProcessTaskType.Executable, ProcessTaskType.SQLBakFile };
if (!AcceptedProcessTaskTypes.Contains(taskType))
SetImpossible("Only SQLFile, SqlBakFile and Executable task types are supported by this command");

if (!ProcessTask.IsCompatibleStage(taskType, loadStage))
SetImpossible($"You cannot run {taskType} in {loadStage}");
Expand All @@ -56,7 +58,26 @@ public override void Execute()

if (_file == null)
{
if (_taskType == ProcessTaskType.SQLFile)
if (_taskType == ProcessTaskType.SQLBakFile)
{
if (BasicActivator.TypeText("Enter a name for the SQL Bak file", "File name", 100, "database.bak",
JFriel marked this conversation as resolved.
Show resolved Hide resolved
out var selected, false))
{
var target = Path.Combine(_loadDirectory.ExecutablesPath.FullName, selected);

if (!File.Exists(target))
{
return; //File doesn't exist
}

_file = new FileInfo(target);
}
else
{
return; //user cancelled
}
}
else if (_taskType == ProcessTaskType.SQLFile)
{
if (BasicActivator.TypeText("Enter a name for the SQL file", "File name", 100, "myscript.sql",
out var selected, false))
Expand Down Expand Up @@ -117,6 +138,7 @@ public override string GetCommandName()
{
ProcessTaskType.Executable => "Add Run .exe File Task",
ProcessTaskType.SQLFile => "Add Run SQL Script Task",
ProcessTaskType.SQLBakFile => "Add SQL backup File Task",
_ => throw new ArgumentOutOfRangeException()
};
}
Expand All @@ -126,6 +148,7 @@ public override Image<Rgba32> GetImage(IIconProvider iconProvider)
return _taskType switch
{
ProcessTaskType.SQLFile => iconProvider.GetImage(RDMPConcept.SQL, OverlayKind.Add),
ProcessTaskType.SQLBakFile => iconProvider.GetImage(RDMPConcept.SQL, OverlayKind.Add), //todo maybe better
ProcessTaskType.Executable => IconOverlayProvider.GetOverlayNoCache(
Image.Load<Rgba32>(CatalogueIcons.Exe), OverlayKind.Add),
_ => null
Expand Down
5 changes: 4 additions & 1 deletion Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,9 @@ public void Check(ICheckNotifier notifier)
case ProcessTaskType.SQLFile:
CheckFileExistenceAndUniqueness(notifier);
CheckForProblemsInSQLFile(notifier);

break;
case ProcessTaskType.SQLBakFile:
CheckFileExistenceAndUniqueness(notifier);
break;
case ProcessTaskType.Attacher:
break;
Expand Down Expand Up @@ -374,6 +376,7 @@ public static bool IsCompatibleStage(ProcessTaskType type, LoadStage stage)
{
ProcessTaskType.Executable => true,
ProcessTaskType.SQLFile => stage != LoadStage.GetFiles,
ProcessTaskType.SQLBakFile => stage != LoadStage.GetFiles,
ProcessTaskType.Attacher => stage == LoadStage.Mounting,
ProcessTaskType.DataProvider => true,
ProcessTaskType.MutilateDataTable => stage != LoadStage.GetFiles,
Expand Down
5 changes: 5 additions & 0 deletions Rdmp.Core/Curation/Data/DataLoad/ProcessTaskType.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ public enum ProcessTaskType
/// </summary>
SQLFile,

/// <summary>
/// ProcessTask is to import a SQL backup file directly to the server
/// </summary>
SQLBakFile,

/// <summary>
/// ProcessTask is to instantiate the IAttacher class Type specified in Path and hydrate its [DemandsInitialization] properties with values matching
/// ProcessTaskArguments and run it in the specified load stage in an AttacherRuntimeTask wrapper.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
// Copyright (c) The University of Dundee 2018-2023
// This file is part of the Research Data Management Platform (RDMP).
// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Data;
using System.IO;
using System.Linq;
using Microsoft.Data.SqlClient;
using MongoDB.Driver.Core.Servers;
using Rdmp.Core.Curation.Data.DataLoad;
using Rdmp.Core.DataFlowPipeline;
using Rdmp.Core.DataLoad.Engine.Job;
using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Arguments;
using Rdmp.Core.DataLoad.Modules.Mutilators;
using Rdmp.Core.ReusableLibraryCode.Checks;
using Rdmp.Core.ReusableLibraryCode.Progress;

namespace Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Runtime;

/// <summary>
/// RuntimeTask that executes a single .bak file specified by the user in a ProcessTask with ProcessTaskType SQLBakFile.
/// </summary>
public class ExecuteSqlBakFileRuntimeTask : RuntimeTask
{
public string Filepath;
private readonly IProcessTask _task;

private LoadStage _loadStage;

public ExecuteSqlBakFileRuntimeTask(IProcessTask task, RuntimeArgumentCollection args) : base(task, args)
{
_task = task;
Filepath = task.Path;
}

public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
{
var db = RuntimeArguments.StageSpecificArguments.DbInfo;
_loadStage = RuntimeArguments.StageSpecificArguments.LoadStage;

if (!Exists())
throw new Exception($"The sql bak file {Filepath} does not exist");

string fileOnlyCommand = $"RESTORE FILELISTONLY FROM DISK = '{Filepath}'";
DataTable fileInfo = new DataTable();
Fixed Show fixed Hide fixed
using var con = (SqlConnection)db.Server.GetConnection();
SqlCommand cmd = new SqlCommand(fileOnlyCommand, con);
Fixed Show fixed Hide fixed
SqlDataAdapter da = new SqlDataAdapter(cmd);
Fixed Show fixed Hide fixed
da.Fill(fileInfo);

DataRow[] primaryFiles = fileInfo.Select("Type = 'D'");
DataRow[] logFiles = fileInfo.Select("Type = 'L'");
fileInfo.Dispose();
github-advanced-security[bot] marked this conversation as resolved.
Fixed
Show resolved Hide resolved
Fixed Show fixed Hide fixed
cmd.Dispose();
Fixed Show fixed Hide fixed
da.Dispose();
Fixed Show fixed Hide fixed
if (primaryFiles.Length != 1 || logFiles.Length != 1)
{
//Something has gone wrong
return ExitCodeType.Error;
}


DataRow primaryFile = primaryFiles[0];
DataRow logFile = logFiles[0];
string primaryFilePhysicalName = primaryFile["PhysicalName"].ToString();
string logFilePhysicalName = logFile["PhysicalName"].ToString();

if (File.Exists(primaryFilePhysicalName) || File.Exists(logFilePhysicalName))
{
string timestamp = DateTime.Now.Millisecond.ToString();
string primaryFileName = primaryFilePhysicalName.Substring(0, primaryFilePhysicalName.Length - 4);
string primaryFileExtention = primaryFilePhysicalName.Substring(primaryFilePhysicalName.Length - 4);
primaryFilePhysicalName = $"{primaryFileName}_{timestamp}{primaryFileExtention}";
string logFileName = logFilePhysicalName.Substring(0, logFilePhysicalName.Length - 4);
string logFileExtention = logFilePhysicalName.Substring(logFilePhysicalName.Length - 4);
logFilePhysicalName = $"{logFileName}_{timestamp}{logFileExtention}";
}

string name = db.ToString();

string restoreCommand = @$"
use master;
ALTER DATABASE {name} SET SINGLE_USER WITH ROLLBACK IMMEDIATE;
RESTORE DATABASE {name}
FROM DISK = '{Filepath}'
WITH MOVE '{primaryFile["LogicalName"]}' TO '{primaryFilePhysicalName}',
MOVE '{logFile["LogicalName"]}' TO '{logFilePhysicalName}' , NOUNLOAD, REPLACE, STATS = 5;
ALTER DATABASE {name} SET MULTI_USER;
";

job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
$"Executing script {Filepath} ( against {db})"));

var executer = new ExecuteSqlInDleStage(job, _loadStage);
return executer.Execute(restoreCommand, db);
}


public override bool Exists() => File.Exists(Filepath);

public override void Abort(IDataLoadEventListener postLoadEventListener)
{
}

public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
{
}

public override void Check(ICheckNotifier notifier)
{
if (string.IsNullOrWhiteSpace(Filepath))
{
notifier.OnCheckPerformed(
new CheckEventArgs($"ExecuteSqlFileTask {_task} does not have a path specified",
CheckResult.Fail));
return;
}

if (!File.Exists(Filepath))
notifier.OnCheckPerformed(
new CheckEventArgs(
$"File '{Filepath}' does not exist! (the only time this would be legal is if you have an exe or a freaky plugin that creates this file)",
CheckResult.Warning));
else
notifier.OnCheckPerformed(new CheckEventArgs($"Found File '{Filepath}'",
CheckResult.Success));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public static RuntimeTask Create(IProcessTask task, IStageArgs stageArgs)
ProcessTaskType.Attacher => new AttacherRuntimeTask(task, args),
ProcessTaskType.DataProvider => new DataProviderRuntimeTask(task, args),
ProcessTaskType.MutilateDataTable => new MutilateDataTablesRuntimeTask(task, args),
ProcessTaskType.SQLBakFile => new ExecuteSqlBakFileRuntimeTask(task, args),
_ => throw new Exception($"Cannot create runtime task: Unknown process task type '{task.ProcessTaskType}'")
};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public Image<Rgba32> GetImageIfSupportedObject(object o)
{
ProcessTaskType.Executable => _exe,
ProcessTaskType.SQLFile => _sql,
ProcessTaskType.SQLBakFile => _sql,
ProcessTaskType.Attacher => _attacher,
ProcessTaskType.DataProvider => _dataProvider,
ProcessTaskType.MutilateDataTable => _mutilateDataTables,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, Lo
case ".sql":
return new ExecuteCommandCreateNewFileBasedProcessTask(ItemActivator, ProcessTaskType.SQLFile,
targetStage.LoadMetadata, targetStage.LoadStage, f);
case ".bak":
return new ExecuteCommandCreateNewFileBasedProcessTask(ItemActivator, ProcessTaskType.SQLBakFile,
targetStage.LoadMetadata, targetStage.LoadStage, f);
case ".exe":
return new ExecuteCommandCreateNewFileBasedProcessTask(ItemActivator, ProcessTaskType.Executable,
targetStage.LoadMetadata, targetStage.LoadStage, f);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ public override void Activate(ProcessTask processTask)
case ProcessTaskType.SQLFile:
ItemActivator.Activate<SqlProcessTaskUI, ProcessTask>(processTask);
break;
case ProcessTaskType.SQLBakFile:
ItemActivator.Activate<SqlProcessTaskUI, ProcessTask>(processTask);
break;
}
}

Expand Down