Enable DECCOLM support via a private mode escape sequence

src/host/ut_host/ScreenBufferTests.cpp

@@ -107,6 +107,7 @@ class ScreenBufferTests
 
     TEST_METHOD(VtResize);
     TEST_METHOD(VtResizeComprehensive);
+    TEST_METHOD(VtResizeDECCOLM);
 
     TEST_METHOD(VtSoftResetCursorPosition);
 
@@ -972,6 +973,137 @@ void ScreenBufferTests::VtResizeComprehensive()
     VERIFY_ARE_EQUAL(expectedViewHeight, newViewHeight);
 }
 
+void ScreenBufferTests::VtResizeDECCOLM()
+{
+    // Run this test in isolation - for one reason or another, this breaks other tests.
+    BEGIN_TEST_METHOD_PROPERTIES()
+        TEST_METHOD_PROPERTY(L"IsolationLevel", L"Method")
+    END_TEST_METHOD_PROPERTIES()
+
+    auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
+    auto& si = gci.GetActiveOutputBuffer().GetActiveBuffer();
+    auto& stateMachine = si.GetStateMachine();
+    WI_SetFlag(si.OutputMode, ENABLE_VIRTUAL_TERMINAL_PROCESSING);
+
+    const auto setInitialMargins = L"\x1b[5;15r";
+    const auto setInitialCursor = L"\x1b[10;40HABCDEF";
+    const auto allowDECCOLM = L"\x1b[?40h";
+    const auto disallowDECCOLM = L"\x1b[?40l";
+    const auto setDECCOLM = L"\x1b[?3h";
+    const auto resetDECCOLM = L"\x1b[?3l";
+
+    auto getRelativeCursorPosition = [&]() {
+        return si.GetTextBuffer().GetCursor().GetPosition() - si.GetViewport().Origin();
+    };
+
+    stateMachine.ProcessString(setInitialMargins);
+    stateMachine.ProcessString(setInitialCursor);
+    auto initialMargins = si.GetRelativeScrollMargins();
+    auto initialCursorPosition = getRelativeCursorPosition();
+
+    auto initialSbHeight = si.GetBufferSize().Height();
+    auto initialSbWidth = si.GetBufferSize().Width();
+    auto initialViewHeight = si.GetViewport().Height();
+    auto initialViewWidth = si.GetViewport().Width();
+
+    Log::Comment(L"By default, setting DECCOLM should have no effect");
+    stateMachine.ProcessString(setDECCOLM);
+
+    auto newSbHeight = si.GetBufferSize().Height();
+    auto newSbWidth = si.GetBufferSize().Width();
+    auto newViewHeight = si.GetViewport().Height();
+    auto newViewWidth = si.GetViewport().Width();
+
+    VERIFY_IS_TRUE(si.AreMarginsSet());
+    VERIFY_ARE_EQUAL(initialMargins, si.GetRelativeScrollMargins());
+    VERIFY_ARE_EQUAL(initialCursorPosition, getRelativeCursorPosition());
+    VERIFY_ARE_EQUAL(initialSbHeight, newSbHeight);
+    VERIFY_ARE_EQUAL(initialViewHeight, newViewHeight);
+    VERIFY_ARE_EQUAL(initialSbWidth, newSbWidth);
+    VERIFY_ARE_EQUAL(initialViewWidth, newViewWidth);
+
+    stateMachine.ProcessString(setInitialMargins);
+    stateMachine.ProcessString(setInitialCursor);
+
+    initialSbHeight = newSbHeight;
+    initialSbWidth = newSbWidth;
+    initialViewHeight = newViewHeight;
+    initialViewWidth = newViewWidth;
+
+    Log::Comment(
+        L"Once DECCOLM is allowed, setting it "
+        L"should change the width to 132 columns "
+        L"and reset the margins and cursor position");
+    stateMachine.ProcessString(allowDECCOLM);
+    stateMachine.ProcessString(setDECCOLM);
+
+    newSbHeight = si.GetBufferSize().Height();
+    newSbWidth = si.GetBufferSize().Width();
+    newViewHeight = si.GetViewport().Height();
+    newViewWidth = si.GetViewport().Width();
+
+    VERIFY_IS_FALSE(si.AreMarginsSet());
+    VERIFY_ARE_EQUAL(COORD({ 0, 0 }), getRelativeCursorPosition());
+    VERIFY_ARE_EQUAL(initialSbHeight, newSbHeight);
+    VERIFY_ARE_EQUAL(initialViewHeight, newViewHeight);
+    VERIFY_ARE_EQUAL(132, newSbWidth);
+    VERIFY_ARE_EQUAL(132, newViewWidth);
+
+    stateMachine.ProcessString(setInitialMargins);
+    stateMachine.ProcessString(setInitialCursor);
+    initialMargins = si.GetRelativeScrollMargins();
+    initialCursorPosition = getRelativeCursorPosition();
+
+    initialSbHeight = newSbHeight;
+    initialSbWidth = newSbWidth;
+    initialViewHeight = newViewHeight;
+    initialViewWidth = newViewWidth;
+
+    Log::Comment(L"If DECCOLM is disallowed, resetting it should have no effect");
+    stateMachine.ProcessString(disallowDECCOLM);
+    stateMachine.ProcessString(resetDECCOLM);
+
+    newSbHeight = si.GetBufferSize().Height();
+    newSbWidth = si.GetBufferSize().Width();
+    newViewHeight = si.GetViewport().Height();
+    newViewWidth = si.GetViewport().Width();
+
+    VERIFY_IS_TRUE(si.AreMarginsSet());
+    VERIFY_ARE_EQUAL(initialMargins, si.GetRelativeScrollMargins());
+    VERIFY_ARE_EQUAL(initialCursorPosition, getRelativeCursorPosition());
+    VERIFY_ARE_EQUAL(initialSbHeight, newSbHeight);
+    VERIFY_ARE_EQUAL(initialViewHeight, newViewHeight);
+    VERIFY_ARE_EQUAL(initialSbWidth, newSbWidth);
+    VERIFY_ARE_EQUAL(initialViewWidth, newViewWidth);
+
+    stateMachine.ProcessString(setInitialMargins);
+    stateMachine.ProcessString(setInitialCursor);
+
+    initialSbHeight = newSbHeight;
+    initialSbWidth = newSbWidth;
+    initialViewHeight = newViewHeight;
+    initialViewWidth = newViewWidth;
+
+    Log::Comment(
+        L"Once DECCOLM is allowed again, resetting it "
+        L"should change the width to 80 columns "
+        L"and reset the margins and cursor position");
+    stateMachine.ProcessString(allowDECCOLM);
+    stateMachine.ProcessString(resetDECCOLM);
+
+    newSbHeight = si.GetBufferSize().Height();
+    newSbWidth = si.GetBufferSize().Width();
+    newViewHeight = si.GetViewport().Height();
+    newViewWidth = si.GetViewport().Width();
+
+    VERIFY_IS_FALSE(si.AreMarginsSet());
+    VERIFY_ARE_EQUAL(COORD({ 0, 0 }), getRelativeCursorPosition());
+    VERIFY_ARE_EQUAL(initialSbHeight, newSbHeight);
+    VERIFY_ARE_EQUAL(initialViewHeight, newViewHeight);
+    VERIFY_ARE_EQUAL(80, newSbWidth);
+    VERIFY_ARE_EQUAL(80, newViewWidth);
+}
+
 void ScreenBufferTests::VtSoftResetCursorPosition()
 {
     CONSOLE_INFORMATION& gci = ServiceLocator::LocateGlobals().getConsoleInformation();

src/terminal/adapter/DispatchTypes.hpp

@@ -77,6 +77,7 @@ namespace Microsoft::Console::VirtualTerminal::DispatchTypes
         DECCOLM_SetNumberOfColumns = 3,
         ATT610_StartCursorBlink = 12,
         DECTCEM_TextCursorEnableMode = 25,
+        XTERM_EnableDECCOLMSupport = 40,
         VT200_MOUSE_MODE = 1000,
         BUTTTON_EVENT_MOUSE_MODE = 1002,
         ANY_EVENT_MOUSE_MODE = 1003,

src/terminal/adapter/ITermDispatch.hpp

@@ -58,6 +58,7 @@ class Microsoft::Console::VirtualTerminal::ITermDispatch
     virtual bool ForwardTab(const SHORT sNumTabs) = 0; // CHT
     virtual bool BackwardsTab(const SHORT sNumTabs) = 0; // CBT
     virtual bool TabClear(const SHORT sClearType) = 0; // TBC
+    virtual bool EnableDECCOLMSupport(const bool fEnabled) = 0; // ?40
     virtual bool EnableVT200MouseMode(const bool fEnabled) = 0; // ?1000
     virtual bool EnableUTF8ExtendedMouseMode(const bool fEnabled) = 0; // ?1005
     virtual bool EnableSGRExtendedMouseMode(const bool fEnabled) = 0; // ?1006

src/terminal/adapter/adaptDispatch.cpp

@@ -39,6 +39,7 @@ AdaptDispatch::AdaptDispatch(ConGetSet* const pConApi,
                              AdaptDefaults* const pDefaults) :
     _conApi{ THROW_IF_NULL_ALLOC(pConApi) },
     _pDefaults{ THROW_IF_NULL_ALLOC(pDefaults) },
+    _fIsDECCOLMAllowed(false), // by default, DECCOLM is not allowed.
     _fChangedBackground(false),
     _fChangedForeground(false),
     _fChangedMetaAttrs(false),
@@ -48,8 +49,6 @@ AdaptDispatch::AdaptDispatch(ConGetSet* const pConApi,
     _coordSavedCursor.X = 1;
     _coordSavedCursor.Y = 1;
     _srScrollMargins = { 0 }; // initially, there are no scroll margins.
-    _fIsSetColumnsEnabled = false; // by default, DECSCPP is disabled.
-    // TODO:10086990 - Create a setting to re-enable this.
 }
 
 void AdaptDispatch::Print(const wchar_t wchPrintable)
@@ -1055,12 +1054,6 @@ bool AdaptDispatch::ScrollDown(_In_ unsigned int const uiDistance)
 // - True if handled successfully. False otherwise.
 bool AdaptDispatch::SetColumns(_In_ unsigned int const uiColumns)
 {
-    if (!_fIsSetColumnsEnabled)
-    {
-        // Only set columns if that option is available. Return true, as this is technically a successful handling.
-        return true;
-    }
-
     SHORT sColumns;
     bool fSuccess = SUCCEEDED(UIntToShort(uiColumns, &sColumns));
     if (fSuccess)
@@ -1086,6 +1079,12 @@ bool AdaptDispatch::SetColumns(_In_ unsigned int const uiColumns)
 // - True if handled successfully. False otherwise.
 bool AdaptDispatch::_DoDECCOLMHelper(_In_ unsigned int const uiColumns)
 {
+    if (!_fIsDECCOLMAllowed)
+    {
+        // Only proceed if DECCOLM is allowed. Return true, as this is technically a successful handling.
+        return true;
+    }
+
     bool fSuccess = SetColumns(uiColumns);
     if (fSuccess)
     {
@@ -1120,6 +1119,9 @@ bool AdaptDispatch::_PrivateModeParamsHelper(_In_ DispatchTypes::PrivateModePara
     case DispatchTypes::PrivateModeParams::DECTCEM_TextCursorEnableMode:
         fSuccess = CursorVisibility(fEnable);
         break;
+    case DispatchTypes::PrivateModeParams::XTERM_EnableDECCOLMSupport:
+        fSuccess = EnableDECCOLMSupport(fEnable);
+        break;
     case DispatchTypes::PrivateModeParams::VT200_MOUSE_MODE:
         fSuccess = EnableVT200MouseMode(fEnable);
         break;
@@ -1679,6 +1681,18 @@ bool AdaptDispatch::_EraseAll()
     return !!_conApi->PrivateEraseAll();
 }
 
+// Routine Description:
+// - Enables or disables support for the DECCOLM escape sequence.
+// Arguments:
+// - fEnabled - set to true to allow DECCOLM to be used, false to disallow.
+// Return Value:
+// - True if handled successfully. False otherwise.
+bool AdaptDispatch::EnableDECCOLMSupport(const bool fEnabled)
+{
+    _fIsDECCOLMAllowed = fEnabled;
+    return true;
+}
+
 //Routine Description:
 // Enable VT200 Mouse Mode - Enables/disables the mouse input handler in default tracking mode.
 //Arguments:

src/terminal/adapter/adaptDispatch.hpp

@@ -85,6 +85,7 @@ namespace Microsoft::Console::VirtualTerminal
         bool DesignateCharset(const wchar_t wchCharset) override; // DesignateCharset
         bool SoftReset() override; // DECSTR
         bool HardReset() override; // RIS
+        bool EnableDECCOLMSupport(const bool fEnabled) override; // ?40
         bool EnableVT200MouseMode(const bool fEnabled) override; // ?1000
         bool EnableUTF8ExtendedMouseMode(const bool fEnabled) override; // ?1005
         bool EnableSGRExtendedMouseMode(const bool fEnabled) override; // ?1006
@@ -149,7 +150,7 @@ namespace Microsoft::Console::VirtualTerminal
         COORD _coordSavedCursor;
         SMALL_RECT _srScrollMargins;
 
-        bool _fIsSetColumnsEnabled;
+        bool _fIsDECCOLMAllowed;
 
         bool _fChangedForeground;
         bool _fChangedBackground;

src/terminal/adapter/termDispatch.hpp

@@ -55,6 +55,7 @@ class Microsoft::Console::VirtualTerminal::TermDispatch : public Microsoft::Cons
     bool ForwardTab(const SHORT /*sNumTabs*/) override { return false; } // CHT
     bool BackwardsTab(const SHORT /*sNumTabs*/) override { return false; } // CBT
     bool TabClear(const SHORT /*sClearType*/) override { return false; } // TBC
+    bool EnableDECCOLMSupport(const bool /*fEnabled*/) override { return false; } // ?40
     bool EnableVT200MouseMode(const bool /*fEnabled*/) override { return false; } // ?1000
     bool EnableUTF8ExtendedMouseMode(const bool /*fEnabled*/) override { return false; } // ?1005
     bool EnableSGRExtendedMouseMode(const bool /*fEnabled*/) override { return false; } // ?1006

src/terminal/parser/ut_parser/OutputEngineTest.cpp

@@ -641,6 +641,7 @@ class StatefulDispatch final : public TermDispatch
         _fIsAltBuffer{ false },
         _fCursorKeysMode{ false },
         _fCursorBlinking{ true },
+        _fIsDECCOLMAllowed{ false },
         _uiWindowWidth{ 80 }
     {
         memset(_rgOptions, s_uiGraphicsCleared, sizeof(_rgOptions));
@@ -807,6 +808,9 @@ class StatefulDispatch final : public TermDispatch
         case DispatchTypes::PrivateModeParams::DECTCEM_TextCursorEnableMode:
             fSuccess = CursorVisibility(fEnable);
             break;
+        case DispatchTypes::PrivateModeParams::XTERM_EnableDECCOLMSupport:
+            fSuccess = EnableDECCOLMSupport(fEnable);
+            break;
         case DispatchTypes::PrivateModeParams::ASB_AlternateScreenBuffer:
             fSuccess = fEnable ? UseAlternateScreenBuffer() : UseMainScreenBuffer();
             break;
@@ -860,6 +864,12 @@ class StatefulDispatch final : public TermDispatch
         return true;
     }
 
+    bool EnableDECCOLMSupport(const bool fEnabled) override
+    {
+        _fIsDECCOLMAllowed = fEnabled;
+        return true;
+    }
+
     bool UseAlternateScreenBuffer() override
     {
         _fIsAltBuffer = true;
@@ -899,6 +909,7 @@ class StatefulDispatch final : public TermDispatch
     bool _fIsAltBuffer;
     bool _fCursorKeysMode;
     bool _fCursorBlinking;
+    bool _fIsDECCOLMAllowed;
     unsigned int _uiWindowWidth;
 
     static const size_t s_cMaxOptions = 16;
@@ -1265,6 +1276,24 @@ class StateMachineExternalTest final
         pDispatch->ClearState();
     }
 
+    TEST_METHOD(TestEnableDECCOLMSupport)
+    {
+        StatefulDispatch* pDispatch = new StatefulDispatch;
+        VERIFY_IS_NOT_NULL(pDispatch);
+        StateMachine mach(new OutputStateMachineEngine(pDispatch));
+
+        mach.ProcessString(L"\x1b[?40h");
+        VERIFY_IS_TRUE(pDispatch->_fIsDECCOLMAllowed);
+
+        pDispatch->ClearState();
+        pDispatch->_fIsDECCOLMAllowed = true;
+
+        mach.ProcessString(L"\x1b[?40l");
+        VERIFY_IS_FALSE(pDispatch->_fIsDECCOLMAllowed);
+
+        pDispatch->ClearState();
+    }
+
     TEST_METHOD(TestErase)
     {
         BEGIN_TEST_METHOD_PROPERTIES()