Remove obsolete handling of \d marker (#216)

- fixes sillsdev/serval#416
- add UsfmMemoryText for testing
- add support for isolated testing of USFM parsing/generation

src/SIL.Machine/Corpora/MemoryStreamContainer.cs

@@ -0,0 +1,21 @@
+using System.IO;
+using System.Text;
+using SIL.ObjectModel;
+
+namespace SIL.Machine.Corpora
+{
+    public class MemoryStreamContainer : DisposableBase, IStreamContainer
+    {
+        private readonly string _usfm;
+
+        public MemoryStreamContainer(string usfm)
+        {
+            _usfm = usfm;
+        }
+
+        public Stream OpenStream()
+        {
+            return new MemoryStream(Encoding.UTF8.GetBytes(_usfm));
+        }
+    }
+}

src/SIL.Machine/Corpora/ScriptureRefUsfmParserHandlerBase.cs

@@ -84,7 +84,7 @@ IReadOnlyList<UsfmAttribute> attributes
             if (_curVerseRef.IsDefault)
                 UpdateVerseRef(state.VerseRef, marker);
 
-            if (!state.IsVerseText || marker == "d")
+            if (!state.IsVerseText)
             {
                 StartParentElement(marker);
                 StartNonVerseText(state);

src/SIL.Machine/Corpora/UsfmMemoryText.cs

@@ -0,0 +1,29 @@
+using System.Text;
+using SIL.Scripture;
+
+namespace SIL.Machine.Corpora
+{
+    public class UsfmMemoryText : UsfmTextBase
+    {
+        private readonly string _usfm;
+
+        public UsfmMemoryText(
+            UsfmStylesheet stylesheet,
+            Encoding encoding,
+            string id,
+            string usfm,
+            ScrVers versification = null,
+            bool includeMarkers = false,
+            bool includeAllText = false
+        )
+            : base(id, stylesheet, encoding, versification, includeMarkers, includeAllText)
+        {
+            _usfm = usfm;
+        }
+
+        protected override IStreamContainer CreateStreamContainer()
+        {
+            return new MemoryStreamContainer(_usfm);
+        }
+    }
+}

tests/SIL.Machine.Tests/Corpora/CorporaTestHelpers.cs

@@ -1,4 +1,5 @@
 using System.IO.Compression;
+using NUnit.Framework.Constraints;
 
 namespace SIL.Machine.Corpora;
 
@@ -35,4 +36,11 @@ public static string CreateTestParatextBackup()
         ZipFile.CreateFromDirectory(UsfmTestProjectPath, path);
         return path;
     }
+
+    public static EqualConstraint IgnoreLineEndings(this EqualConstraint constraint)
+    {
+        return constraint.Using<string>(
+            (actual, expected) => actual.ReplaceLineEndings() == expected.ReplaceLineEndings()
+        );
+    }
 }

tests/SIL.Machine.Tests/Corpora/UsfmMemoryTextTests.cs

@@ -0,0 +1,65 @@
+using System.Text;
+using NUnit.Framework;
+
+namespace SIL.Machine.Corpora;
+
+[TestFixture]
+public class UsfmMemoryTextTests
+{
+    [Test]
+    public void GetRows_VerseDescriptiveTitle()
+    {
+        TextRow[] rows = GetRows(
+            @"\id MAT - Test
+\c 1
+\d
+\v 1 Descriptive title
+\c 2
+\b
+\q1
+\s
+"
+        );
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(rows, Has.Length.EqualTo(1));
+
+            Assert.That(rows[0].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 1:1")));
+            Assert.That(rows[0].Text, Is.EqualTo("Descriptive title"));
+        });
+    }
+
+    [Test]
+    public void GetRows_LastSegment()
+    {
+        TextRow[] rows = GetRows(
+            @"\id MAT - Test
+\c 1
+\v 1 Last segment
+"
+        );
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(rows, Has.Length.EqualTo(1));
+
+            Assert.That(rows[0].Ref, Is.EqualTo(ScriptureRef.Parse("MAT 1:1")));
+            Assert.That(rows[0].Text, Is.EqualTo("Last segment"));
+        });
+    }
+
+    private static TextRow[] GetRows(string usfm, bool includeMarkers = false, bool includeAllText = false)
+    {
+        UsfmMemoryText text =
+            new(
+                new UsfmStylesheet("usfm.sty"),
+                Encoding.UTF8,
+                "MAT",
+                usfm.Trim().ReplaceLineEndings("\r\n") + "\r\n",
+                includeMarkers: includeMarkers,
+                includeAllText: includeAllText
+            );
+        return text.GetRows().ToArray();
+    }
+}

tests/SIL.Machine.Tests/Corpora/UsfmTextUpdaterTests.cs

@@ -376,13 +376,26 @@ public void GetUsfm_Verse_DoubleVaVp()
     }
 
     [Test]
-    public void GetUsfm_Verse_LastVerse()
+    public void GetUsfm_Verse_LastSegment()
     {
-        var rows = new List<(IReadOnlyList<ScriptureRef>, string)> { (ScrRef("MAT 4:1"), "Updating the last verse.") };
+        var rows = new List<(IReadOnlyList<ScriptureRef>, string)> { (ScrRef("MAT 1:1"), "Updating the last verse.") };
+        string usfm =
+            @"\id MAT - Test
+\c 1
+\v 1
+";
+        string target = UpdateUsfm(rows, usfm);
 
-        string target = UpdateUsfm(rows);
-        Assert.That(target, Contains.Substring("\\id MAT - Test\r\n"));
-        Assert.That(target, Contains.Substring("\\v 1 Updating the last verse.\r\n"));
+        Assert.That(
+            target,
+            Is.EqualTo(
+                    @"\id MAT - Test
+\c 1
+\v 1 Updating the last verse.
+"
+                )
+                .IgnoreLineEndings()
+        );
     }
 
     [Test]
@@ -409,12 +422,16 @@ private static ScriptureRef[] ScrRef(params string[] refs)
 
     private static string UpdateUsfm(
         IReadOnlyList<(IReadOnlyList<ScriptureRef>, string)>? rows = null,
+        string? source = null,
         string? idText = null,
         bool stripAllText = false,
         bool preferExistingText = false
     )
     {
-        string source = ReadUsfm();
+        if (source is null)
+            source = ReadUsfm();
+        else
+            source = source.Trim().ReplaceLineEndings("\r\n") + "\r\n";
         var updater = new UsfmTextUpdater(
             rows,
             idText,