From f3092e14a2bad246764241eabe585c22b334765b Mon Sep 17 00:00:00 2001 From: Alexander Hjelm Date: Mon, 16 Dec 2024 10:34:49 +0100 Subject: [PATCH] Add option to defer ExecutionItems until later. --- .../AttachmentNotFoundException.cs | 21 +++++++ src/WorkItemMigrator/WorkItemImport/Agent.cs | 4 ++ .../WorkItemImport/ExecutionPlan.cs | 13 +++++ .../WorkItemImport/ImportCommandLine.cs | 58 ++++++++++++++++++- .../WitClient/WitClientUtils.cs | 5 +- 5 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 src/WorkItemMigrator/Migration.Common.Log/AttachmentNotFoundException.cs diff --git a/src/WorkItemMigrator/Migration.Common.Log/AttachmentNotFoundException.cs b/src/WorkItemMigrator/Migration.Common.Log/AttachmentNotFoundException.cs new file mode 100644 index 00000000..d9a01ea0 --- /dev/null +++ b/src/WorkItemMigrator/Migration.Common.Log/AttachmentNotFoundException.cs @@ -0,0 +1,21 @@ +using System; +using System.Runtime.Serialization; + +namespace Migration.Common.Log +{ + [Serializable] + public class AttachmentNotFoundException : Exception + { + protected AttachmentNotFoundException(SerializationInfo serializationInfo, StreamingContext streamingContext) : base(serializationInfo, streamingContext) + { + + } + + public AttachmentNotFoundException(string reason) + { + Reason = reason; + } + + public string Reason { get; private set; } + } +} \ No newline at end of file diff --git a/src/WorkItemMigrator/WorkItemImport/Agent.cs b/src/WorkItemMigrator/WorkItemImport/Agent.cs index 09cab1af..991a3d8a 100644 --- a/src/WorkItemMigrator/WorkItemImport/Agent.cs +++ b/src/WorkItemMigrator/WorkItemImport/Agent.cs @@ -190,6 +190,10 @@ public bool ImportRevision(WiRevision rev, WorkItem wi, Settings settings) { throw; } + catch (AttachmentNotFoundException) + { + throw; + } catch (FileNotFoundException ex) { Logger.Log(LogLevel.Error, ex.Message); diff --git a/src/WorkItemMigrator/WorkItemImport/ExecutionPlan.cs b/src/WorkItemMigrator/WorkItemImport/ExecutionPlan.cs index 43b2b7c6..a5dd1ebc 100644 --- a/src/WorkItemMigrator/WorkItemImport/ExecutionPlan.cs +++ b/src/WorkItemMigrator/WorkItemImport/ExecutionPlan.cs @@ -12,6 +12,7 @@ public class ExecutionItem public int WiId { get; set; } = -1; public WiRevision Revision { get; set; } public string WiType { get; internal set; } + public bool isDeferred { get; set; } public override string ToString() { @@ -48,5 +49,17 @@ public bool TryPop(out ExecutionItem nextItem) else return false; } + + public bool TryPeek(out ExecutionItem nextItem) + { + nextItem = null; + if (ReferenceQueue.Count > 0) + { + nextItem = TransformToExecutionItem(ReferenceQueue.Peek()); + return true; + } + else + return false; + } } } diff --git a/src/WorkItemMigrator/WorkItemImport/ImportCommandLine.cs b/src/WorkItemMigrator/WorkItemImport/ImportCommandLine.cs index b05b34d6..f216e553 100644 --- a/src/WorkItemMigrator/WorkItemImport/ImportCommandLine.cs +++ b/src/WorkItemMigrator/WorkItemImport/ImportCommandLine.cs @@ -16,6 +16,7 @@ public class ImportCommandLine { private CommandLineApplication commandLineApplication; private string[] args; + private List deferredExecutionItems = new List(); public ImportCommandLine(params string[] args) { @@ -112,10 +113,18 @@ private bool ExecuteMigration(CommandOption token, CommandOption url, CommandOpt BeginSession(configFileName, config, forceFresh, agent, itemCount, revisionCount); - while (plan.TryPop(out ExecutionPlan.ExecutionItem executionItem)) + while (plan.ReferenceQueue.Count > 0 || deferredExecutionItems.Count > 0) { + ExecutionPlan.ExecutionItem executionItem = null; try { + executionItem = GetDeferredItemIfAvailable(plan, executionItem); + + if (executionItem == null) + { + plan.TryPop(out executionItem); + } + if (!forceFresh && context.Journal.IsItemMigrated(executionItem.OriginId, executionItem.Revision.Index)) { continue; @@ -156,7 +165,14 @@ private bool ExecuteMigration(CommandOption token, CommandOption url, CommandOpt continue; } - agent.ImportRevision(executionItem.Revision, wi, settings); + try + { + agent.ImportRevision(executionItem.Revision, wi, settings); + } + catch (AttachmentNotFoundException) + { + importedItems = DeferItem(importedItems, executionItem); + } // Artifical wait (optional) to avoid throttling for ADO Services if (config.SleepTimeBetweenRevisionImportMilliseconds > 0) @@ -199,6 +215,44 @@ private bool ExecuteMigration(CommandOption token, CommandOption url, CommandOpt return succeeded; } + private int DeferItem(int importedItems, ExecutionPlan.ExecutionItem executionItem) + { + if (!executionItem.isDeferred) + { + executionItem.Revision.Time = executionItem.Revision.Time.AddMinutes(5); + executionItem.isDeferred = true; + deferredExecutionItems.Add(executionItem); + importedItems--; + } + + return importedItems; + } + + private ExecutionPlan.ExecutionItem GetDeferredItemIfAvailable(ExecutionPlan plan, ExecutionPlan.ExecutionItem executionItem) + { + foreach (var executionItemDeferred in deferredExecutionItems) + { + if (plan.TryPeek(out var nextItem)) + { + if (executionItemDeferred.Revision.Time < nextItem.Revision.Time) + { + executionItem = executionItemDeferred; + deferredExecutionItems.Remove(executionItem); + executionItem.Revision.Time = nextItem.Revision.Time.AddMilliseconds(-5); + break; + } + } + else + { + executionItem = executionItemDeferred; + deferredExecutionItems.Remove(executionItem); + break; + } + } + + return executionItem; + } + private static void BeginSession(string configFile, ConfigJson config, bool force, Agent agent, int itemsCount, int revisionCount) { var toolVersion = VersionInfo.GetVersionInfo(); diff --git a/src/WorkItemMigrator/WorkItemImport/WitClient/WitClientUtils.cs b/src/WorkItemMigrator/WorkItemImport/WitClient/WitClientUtils.cs index d5055f11..bf4eec6b 100644 --- a/src/WorkItemMigrator/WorkItemImport/WitClient/WitClientUtils.cs +++ b/src/WorkItemMigrator/WorkItemImport/WitClient/WitClientUtils.cs @@ -771,7 +771,10 @@ private void CorrectImagePath(WorkItem wi, WiItem wiItem, WiRevision rev, ref st isUpdated = true; } else - Logger.Log(LogLevel.Warning, $"Attachment '{att}' referenced in text but is missing from work item {wiItem.OriginId}/{wi.Id}."); + { + Logger.Log(LogLevel.Warning, $"Attachment '{att}' referenced in text but is missing from work item {wiItem.OriginId}/{wi.Id}. This revision will be deferred until later."); + throw new AttachmentNotFoundException("Attachment not found on work item"); + } } } if (isUpdated)