Add an efficient text stream write function (#14821)

This adds PR adds a couple foundational functions and classes to make
our TextBuffer more performant and allow us to improve our Unicode
correctness in the future, by getting rid of our dependence on
`OutputCellIterator`. In the future we can then replace the simple
UTF-16 code point iterator with a proper grapheme cluster iterator.

While my focus is technically on Unicode correctness, the ~4x VT
throughput increase in OpenConsole is pretty nice too.

This PR adds:
* A new, simpler ROW iterator (unused in this PR)
* Cursor movement functions (`NavigateToPrevious`, `NavigateToNext`)
  They're based on functions that align the cursor to the start/end
  of the _current_ cell, so such functions can be added as well.
* `ReplaceText` to write a raw string of text with the possibility to
  specify a right margin.
* `CopyRangeFrom` will allow us to make reflow much faster, as it's able
  to bulk-copy already measured strings without re-measuring them.

Related to #8000

## Validation Steps Performed
* enwik8.txt, zhwik8.txt, emoji-test.txt, all work with proper
  wide glyph reflow at the end of a row ✅
* This produces "a 咪" where only "a" has a white background:
  ```sh
  printf '\e7こん\e8\x1b[107ma\x1b[m\n'
  ```
* This produces "abん":
  ```sh
  stdbuf -o0 printf '\x1b7こん\x1b8a'; printf 'b\n'
  ```
* This produces "xy" at the end of the line:
  ```sh
  stdbuf -o0 printf '\e[999C\bこ\bx'; printf 'y\n'
  ```
* This produces red whitespace followed by "こ " in the default
  background color at the end of the line, and "ん" on the next line:
  ```sh
  printf '\e[41m\e[K\e[m\e[999C\e[2Dこん\n'
  ```

Author: lhecker

.github/actions/spelling/expect/alphabet.txt

@@ -18,6 +18,7 @@ BBBBBBBB
 BBBBBCCC
 BBBBCCCCC
 BBGGRR
+efg
 EFG
 EFGh
 QQQQQQQQQQABCDEFGHIJ

.github/actions/spelling/expect/expect.txt

@@ -2280,8 +2280,6 @@ xunit
 xutr
 XVIRTUALSCREEN
 XWalk
-xwwyzz
-xxyyzz
 yact
 YCast
 YCENTER

src/buffer/out/Row.cpp

@@ -20,8 +20,6 @@ Revision History:
 
 #pragma once
 
-#include <span>
-
 #include <til/rle.h>
 
 #include "LineRendition.hpp"
@@ -37,6 +35,28 @@ enum class DelimiterClass
     RegularChar
 };
 
+struct RowWriteState
+{
+    // The text you want to write into the given ROW. When ReplaceText() returns,
+    // this is updated to remove all text from the beginning that was successfully written.
+    std::wstring_view text; // IN/OUT
+    // The column at which to start writing.
+    til::CoordType columnBegin = 0; // IN
+    // The first column which should not be written to anymore.
+    til::CoordType columnLimit = 0; // IN
+
+    // The column 1 past the last glyph that was successfully written into the row. If you need to call
+    // ReplaceAttributes() to colorize the written range, etc., this is the columnEnd parameter you want.
+    // If you want to continue writing where you left off, this is also the next columnBegin parameter.
+    til::CoordType columnEnd = 0; // OUT
+    // The first column that got modified by this write operation. In case that the first glyph we write overwrites
+    // the trailing half of a wide glyph, leadingSpaces will be 1 and this value will be 1 less than colBeg.
+    til::CoordType columnBeginDirty = 0; // OUT
+    // This is 1 past the last column that was modified and will be 1 past columnEnd if we overwrote
+    // the leading half of a wide glyph and had to fill the trailing half with whitespace.
+    til::CoordType columnEndDirty = 0; // OUT
+};
+
 class ROW final
 {
 public:
@@ -62,16 +82,23 @@ class ROW final
     void Resize(wchar_t* charsBuffer, uint16_t* charOffsetsBuffer, uint16_t rowWidth, const TextAttribute& fillAttribute);
     void TransferAttributes(const til::small_rle<TextAttribute, uint16_t, 1>& attr, til::CoordType newWidth);
 
+    til::CoordType NavigateToPrevious(til::CoordType column) const noexcept;
+    til::CoordType NavigateToNext(til::CoordType column) const noexcept;
+
     void ClearCell(til::CoordType column);
     OutputCellIterator WriteCells(OutputCellIterator it, til::CoordType columnBegin, std::optional<bool> wrap = std::nullopt, std::optional<til::CoordType> limitRight = std::nullopt);
     bool SetAttrToEnd(til::CoordType columnBegin, TextAttribute attr);
     void ReplaceAttributes(til::CoordType beginIndex, til::CoordType endIndex, const TextAttribute& newAttr);
     void ReplaceCharacters(til::CoordType columnBegin, til::CoordType width, const std::wstring_view& chars);
+    void ReplaceText(RowWriteState& state);
+    til::CoordType CopyRangeFrom(til::CoordType columnBegin, til::CoordType columnLimit, const ROW& other, til::CoordType& otherBegin, til::CoordType otherLimit);
 
+    til::small_rle<TextAttribute, uint16_t, 1>& Attributes() noexcept;
     const til::small_rle<TextAttribute, uint16_t, 1>& Attributes() const noexcept;
     TextAttribute GetAttrByColumn(til::CoordType column) const;
     std::vector<uint16_t> GetHyperlinks() const;
     uint16_t size() const noexcept;
+    til::CoordType LineRenditionColumns() const noexcept;
     til::CoordType MeasureLeft() const noexcept;
     til::CoordType MeasureRight() const noexcept;
     bool ContainsText() const noexcept;
@@ -89,6 +116,50 @@ class ROW final
 #endif
 
 private:
+    // WriteHelper exists because other forms of abstracting this functionality away (like templates with lambdas)
+    // where only very poorly optimized by MSVC as it failed to inline the templates.
+    struct WriteHelper
+    {
+        explicit WriteHelper(ROW& row, til::CoordType columnBegin, til::CoordType columnLimit, const std::wstring_view& chars) noexcept;
+        bool IsValid() const noexcept;
+        void ReplaceCharacters(til::CoordType width) noexcept;
+        void ReplaceText() noexcept;
+        void CopyRangeFrom(const std::span<const uint16_t>& charOffsets) noexcept;
+        void Finish();
+
+        // Parent pointer.
+        ROW& row;
+        // The text given by the caller.
+        const std::wstring_view& chars;
+
+        // This is the same as the columnBegin parameter for ReplaceText(), etc.,
+        // but clamped to a valid range via _clampedColumnInclusive.
+        uint16_t colBeg;
+        // This is the same as the columnLimit parameter for ReplaceText(), etc.,
+        // but clamped to a valid range via _clampedColumnInclusive.
+        uint16_t colLimit;
+
+        // The column 1 past the last glyph that was successfully written into the row. If you need to call
+        // ReplaceAttributes() to colorize the written range, etc., this is the columnEnd parameter you want.
+        // If you want to continue writing where you left off, this is also the next columnBegin parameter.
+        uint16_t colEnd;
+        // The first column that got modified by this write operation. In case that the first glyph we write overwrites
+        // the trailing half of a wide glyph, leadingSpaces will be 1 and this value will be 1 less than colBeg.
+        uint16_t colBegDirty;
+        // Similar to dirtyBeg, this is 1 past the last column that was modified and will be 1 past colEnd if
+        // we overwrote the leading half of a wide glyph and had to fill the trailing half with whitespace.
+        uint16_t colEndDirty;
+        // The offset in ROW::chars at which we start writing the contents of WriteHelper::chars.
+        uint16_t chBeg;
+        // The offset at which we start writing leadingSpaces-many whitespaces.
+        uint16_t chBegDirty;
+        // The same as `colBeg - colBegDirty`. This is the amount of whitespace
+        // we write at chBegDirty, before the actual WriteHelper::chars content.
+        uint16_t leadingSpaces;
+        // The amount of characters copied from WriteHelper::chars.
+        size_t charsConsumed;
+    };
+
     // To simplify the detection of wide glyphs, we don't just store the simple character offset as described
     // for _charOffsets. Instead we use the most significant bit to indicate whether any column is the
     // trailing half of a wide glyph. This simplifies many implementation details via _uncheckedIsTrailer.
@@ -102,13 +173,16 @@ class ROW final
     template<typename T>
     constexpr uint16_t _clampedColumnInclusive(T v) const noexcept;
 
+    uint16_t _adjustBackward(uint16_t column) const noexcept;
+    uint16_t _adjustForward(uint16_t column) const noexcept;
+
     wchar_t _uncheckedChar(size_t off) const noexcept;
     uint16_t _charSize() const noexcept;
     uint16_t _uncheckedCharOffset(size_t col) const noexcept;
     bool _uncheckedIsTrailer(size_t col) const noexcept;
 
     void _init() noexcept;
-    void _resizeChars(uint16_t colExtEnd, uint16_t chExtBeg, uint16_t chExtEnd, size_t chExtEndNew);
+    void _resizeChars(uint16_t colEndDirty, uint16_t chBegDirty, size_t chEndDirty, uint16_t chEndDirtyOld);
 
     // These fields are a bit "wasteful", but it makes all this a bit more robust against
     // programming errors during initial development (which is when this comment was written).

src/buffer/out/Row.hpp

@@ -1,41 +1,7 @@
-/*++
-Copyright (c) Microsoft Corporation
-Licensed under the MIT license.
-
-Module Name:
-- precomp.h
-
-Abstract:
-- Contains external headers to include in the precompile phase of console build process.
-- Avoid including internal project headers. Instead include them only in the classes that need them (helps with test project building).
---*/
-
-// stdafx.h : include file for standard system include files,
-// or project specific include files that are used frequently, but
-// are changed infrequently
-//
-
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
 #pragma once
 
-// clang-format off
-
-// This includes support libraries from the CRT, STL, WIL, and GSL
-#include "LibraryIncludes.h"
-
-#pragma warning(push)
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN             // Exclude rarely-used stuff from Windows headers
-#define NOMCX
-#define NOHELP
-#define NOCOMM
-#endif
-
-// Windows Header Files:
-#include <windows.h>
-#include <intsafe.h>
-
-// private dependencies
-#include "../inc/unicode.hpp"
-#pragma warning(pop)
+#include <LibraryIncludes.h>
 
-// clang-format on
+#include <unicode.hpp>

src/buffer/out/precomp.h

@@ -376,6 +376,32 @@ bool TextBuffer::_PrepareForDoubleByteSequence(const DbcsAttribute dbcsAttribute
     return fSuccess;
 }
 
+void TextBuffer::ConsumeGrapheme(std::wstring_view& chars) noexcept
+{
+    // This function is supposed to mirror the behavior of ROW::Write, when it reads characters off of `chars`.
+    // (I know that a UTF-16 code point is not a grapheme, but that's what we're working towards.)
+    chars = til::utf16_pop(chars);
+}
+
+// This function is intended for writing regular "lines" of text and only the `state.text` and`state.columnBegin`
+// fields are being used, whereas `state.columnLimit` is automatically overwritten by the line width of the given row.
+// This allows this function to automatically set the wrap-forced field of the row, which is also the return value.
+// The return value indicates to the caller whether the cursor should be moved to the next line.
+void TextBuffer::WriteLine(til::CoordType row, bool wrapAtEOL, const TextAttribute& attributes, RowWriteState& state)
+{
+    auto& r = GetRowByOffset(row);
+
+    r.ReplaceText(state);
+    r.ReplaceAttributes(state.columnBegin, state.columnEnd, attributes);
+
+    if (state.columnEnd >= state.columnLimit)
+    {
+        r.SetWrapForced(wrapAtEOL);
+    }
+
+    TriggerRedraw(Viewport::FromExclusive({ state.columnBeginDirty, row, state.columnEndDirty, row + 1 }));
+}
+
 // Routine Description:
 // - Writes cells to the output buffer. Writes at the cursor.
 // Arguments:

src/buffer/out/textBuffer.cpp

@@ -89,6 +89,9 @@ class TextBuffer final
     TextBufferTextIterator GetTextDataAt(const til::point at, const Microsoft::Console::Types::Viewport limit) const;
 
     // Text insertion functions
+    static void ConsumeGrapheme(std::wstring_view& chars) noexcept;
+    void WriteLine(til::CoordType row, bool wrapAtEOL, const TextAttribute& attributes, RowWriteState& state);
+
     OutputCellIterator Write(const OutputCellIterator givenIt);
 
     OutputCellIterator Write(const OutputCellIterator givenIt,

src/buffer/out/textBuffer.hpp

@@ -18,7 +18,7 @@ Author(s):
 // Manually include til after we include Windows.Foundation to give it winrt superpowers
 #define BLOCK_TIL
 // This includes support libraries from the CRT, STL, WIL, and GSL
-#include "LibraryIncludes.h"
+#include <LibraryIncludes.h>
 // This is inexplicable, but for whatever reason, cppwinrt conflicts with the
 //      SDK definition of this function, so the only fix is to undef it.
 // from WinBase.h
@@ -28,8 +28,9 @@ Author(s):
 #endif
 
 #include <wil/cppwinrt.h>
-#include <unknwn.h>
+#include <Unknwn.h>
 #include <hstring.h>
+#include <shellapi.h>
 
 #include <WexTestClass.h>
 #include <json.h>

src/cascadia/LocalTests_SettingsModel/pch.h

@@ -5249,6 +5249,16 @@ void ScreenBufferTests::SetAutoWrapMode()
     // Content should be clamped to the line width, overwriting the last char.
     VERIFY_IS_TRUE(_ValidateLineContains({ 80 - 3, startLine }, L"abf", attributes));
     VERIFY_ARE_EQUAL(til::point(79, startLine), cursor.GetPosition());
+    // Writing a wide glyph into the last 2 columns and overwriting it with a narrow one.
+    cursor.SetPosition({ 80 - 3, startLine });
+    stateMachine.ProcessString(L"a\U0001F604b");
+    VERIFY_IS_TRUE(_ValidateLineContains({ 80 - 3, startLine }, L"a b", attributes));
+    VERIFY_ARE_EQUAL(til::point(79, startLine), cursor.GetPosition());
+    // Writing a wide glyph into the last column and overwriting it with a narrow one.
+    cursor.SetPosition({ 80 - 3, startLine });
+    stateMachine.ProcessString(L"ab\U0001F604c");
+    VERIFY_IS_TRUE(_ValidateLineContains({ 80 - 3, startLine }, L"abc", attributes));
+    VERIFY_ARE_EQUAL(til::point(79, startLine), cursor.GetPosition());
 
     Log::Comment(L"When DECAWM is set, output is wrapped again.");
     stateMachine.ProcessString(L"\x1b[?7h");

src/host/ut_host/ScreenBufferTests.cpp

@@ -147,6 +147,7 @@ class TextBufferTests
 
     TEST_METHOD(TestBurrito);
     TEST_METHOD(TestOverwriteChars);
+    TEST_METHOD(TestRowReplaceText);
 
     TEST_METHOD(TestAppendRTFText);
 
@@ -2046,6 +2047,87 @@ void TextBufferTests::TestOverwriteChars()
 #undef complex1
 }
 
+void TextBufferTests::TestRowReplaceText()
+{
+    static constexpr til::size bufferSize{ 10, 3 };
+    static constexpr UINT cursorSize = 12;
+    const TextAttribute attr{ 0x7f };
+    TextBuffer buffer{ bufferSize, attr, cursorSize, false, _renderer };
+    auto& row = buffer.GetRowByOffset(0);
+
+#define complex L"\U0001F41B"
+
+    struct Test
+    {
+        const wchar_t* description;
+        struct
+        {
+            std::wstring_view text;
+            til::CoordType columnBegin = 0;
+            til::CoordType columnLimit = 0;
+        } input;
+        struct
+        {
+            std::wstring_view text;
+            til::CoordType columnEnd = 0;
+            til::CoordType columnBeginDirty = 0;
+            til::CoordType columnEndDirty = 0;
+        } expected;
+        std::wstring_view expectedRow;
+    };
+
+    static constexpr std::array tests{
+        Test{
+            L"Not enough space -> early exit",
+            { complex, 2, 2 },
+            { complex, 2, 2, 2 },
+            L"          ",
+        },
+        Test{
+            L"Exact right amount of space",
+            { complex, 2, 4 },
+            { L"", 4, 2, 4 },
+            L"  " complex L"      ",
+        },
+        Test{
+            L"Not enough space -> columnEnd = columnLimit",
+            { complex complex, 0, 3 },
+            { complex, 3, 0, 4 },
+            complex L"        ",
+        },
+        Test{
+            L"Too much to fit into the row",
+            { complex L"b" complex L"c" complex L"abcd", 0, til::CoordTypeMax },
+            { L"cd", 10, 0, 10 },
+            complex L"b" complex L"c" complex L"ab",
+        },
+        Test{
+            L"Overwriting wide glyphs dirties both cells, but leaves columnEnd at the end of the text",
+            { L"efg", 1, til::CoordTypeMax },
+            { L"", 4, 0, 5 },
+            L" efg c" complex L"ab",
+        },
+    };
+
+    for (const auto& t : tests)
+    {
+        Log::Comment(t.description);
+        RowWriteState actual{
+            .text = t.input.text,
+            .columnBegin = t.input.columnBegin,
+            .columnLimit = t.input.columnLimit,
+        };
+        row.ReplaceText(actual);
+        VERIFY_ARE_EQUAL(t.expected.text, actual.text);
+        VERIFY_ARE_EQUAL(t.expected.columnEnd, actual.columnEnd);
+        VERIFY_ARE_EQUAL(t.expected.columnBeginDirty, actual.columnBeginDirty);
+        VERIFY_ARE_EQUAL(t.expected.columnEndDirty, actual.columnEndDirty);
+        VERIFY_ARE_EQUAL(t.expectedRow, row.GetText());
+    }
+
+#undef complex
+}
+
 void TextBufferTests::TestAppendRTFText()
 {
     {