Compensate for non-minimal UTF-8 encodings

src/host/ut_host/Utf8ToWideCharParserTests.cpp

@@ -264,6 +264,53 @@ class Utf8ToWideCharParserTests
         }
     }
 
+    TEST_METHOD(NonMinimalFormTest)
+    {
+        Log::Comment(L"Testing that non-minimal forms of a character are tolerated don't stop the rest");
+
+        // clang-format off
+
+        // Test data
+        const unsigned char data[] = {
+            0x60, 0x12, 0x08, 0x7f, // single byte points
+            0xc0, 0x80, // U+0000 as a 2-byte sequence (non-minimal)
+            0x41, 0x48, 0x06, 0x55, // more single byte points
+            0xe0, 0x80, 0x80, // U+0000 as a 3-byte sequence (non-minimal)
+            0x18, 0x77, 0x40, 0x31, // more single byte points
+            0xf0, 0x80, 0x80, 0x80, // U+0000 as a 4-byte sequence (non-minimal)
+            0x59, 0x1f, 0x68, 0x20 // more single byte points
+        };
+
+        // Expected conversion
+        const wchar_t wideData[] = {
+            0x0060, 0x0012, 0x0008, 0x007f,
+            0xfffd, 0xfffd, // The number of replacements per invalid sequence is not intended to be load-bearing
+            0x0041, 0x0048, 0x0006, 0x0055,
+            0xfffd, 0xfffd, // It is just representative of what it looked like when fixing this for GH#3380
+            0x0018, 0x0077, 0x0040, 0x0031,
+            0xfffd, 0xfffd, 0xfffd, // Change if necessary when completing GH#3378
+            0x0059, 0x001f, 0x0068, 0x0020
+        };
+
+        // clang-format on
+
+        const unsigned int count = gsl::narrow_cast<unsigned int>(ARRAYSIZE(data));
+        const unsigned int wideCount = gsl::narrow_cast<unsigned int>(ARRAYSIZE(wideData));
+        unsigned int consumed = 0;
+        unsigned int generated = 0;
+        unique_ptr<wchar_t[]> output{ nullptr };
+        auto parser = Utf8ToWideCharParser{ utf8CodePage };
+
+        VERIFY_SUCCEEDED(parser.Parse(data, count, consumed, output, generated));
+        VERIFY_ARE_EQUAL(count, consumed);
+        VERIFY_ARE_EQUAL(wideCount, generated);
+        VERIFY_IS_NOT_NULL(output.get());
+
+        const auto expected = WEX::Common::String(wideData, wideCount);
+        const auto actual = WEX::Common::String(output.get(), generated);
+        VERIFY_ARE_EQUAL(expected, actual);
+    }
+
     TEST_METHOD(PartialBytesAreDroppedOnCodePageChangeTest)
     {
         Log::Comment(L"Testing that a saved partial sequence is cleared when the codepage changes");

src/host/utf8ToWideCharParser.cpp

@@ -377,8 +377,17 @@ unsigned int Utf8ToWideCharParser::_InvolvedParse(_In_reads_(cb) const byte* con
         _currentState = _State::AwaitingMoreBytes;
         return 0;
     }
+
+    // By this point, all obviously invalid sequences have been removed.
+    // But non-minimal forms of sequences might still exist.
+    // MB2WC will fail non-minimal forms with MB_ERR_INVALID_CHARS at this point.
+    // So we call with flags = 0 such that non-minimal forms get the U+FFFD
+    // replacement character treatment.
+    // This issue and related concerns are fully captured in future work item GH#3378
+    // for future cleanup and reconciliation.
+    // The original issue introducing this was GH#3320.
     int bufferSize = MultiByteToWideChar(_currentCodePage,
-                                         MB_ERR_INVALID_CHARS,
+                                         0,
                                          reinterpret_cast<LPCCH>(validSequence.first.get()),
                                          validSequence.second,
                                          nullptr,