Use UIA notifications for text output

.github/actions/spelling/expect/expect.txt

@@ -2075,6 +2075,7 @@ rxvt
 safearray
 SAFECAST
 safemath
+sapi
 sba
 SBCS
 SBCSDBCS

src/cascadia/TerminalControl/InteractivityAutomationPeer.cpp

@@ -93,6 +93,11 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         _CursorChangedHandlers(*this, nullptr);
     }
 
+    void InteractivityAutomationPeer::NotifyNewOutput(std::wstring_view newOutput)
+    {
+        _NewOutputHandlers(*this, hstring{ newOutput });
+    }
+
 #pragma region ITextProvider
     com_array<XamlAutomation::ITextRangeProvider> InteractivityAutomationPeer::GetSelection()
     {

src/cascadia/TerminalControl/InteractivityAutomationPeer.h

@@ -49,6 +49,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         void SignalSelectionChanged() override;
         void SignalTextChanged() override;
         void SignalCursorChanged() override;
+        void NotifyNewOutput(std::wstring_view newOutput) override;
 #pragma endregion
 
 #pragma region ITextProvider Pattern
@@ -73,6 +74,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         TYPED_EVENT(SelectionChanged, IInspectable, IInspectable);
         TYPED_EVENT(TextChanged, IInspectable, IInspectable);
         TYPED_EVENT(CursorChanged, IInspectable, IInspectable);
+        TYPED_EVENT(NewOutput, IInspectable, hstring);
 
     private:
         Windows::UI::Xaml::Automation::Provider::ITextRangeProvider _CreateXamlUiaTextRange(::ITextRangeProvider* returnVal) const;

src/cascadia/TerminalControl/InteractivityAutomationPeer.idl

@@ -14,5 +14,6 @@ namespace Microsoft.Terminal.Control
         event Windows.Foundation.TypedEventHandler<Object, Object> SelectionChanged;
         event Windows.Foundation.TypedEventHandler<Object, Object> TextChanged;
         event Windows.Foundation.TypedEventHandler<Object, Object> CursorChanged;
+        event Windows.Foundation.TypedEventHandler<Object, String> NewOutput;
     }
 }

src/cascadia/TerminalControl/TermControl.cpp

@@ -1097,6 +1097,11 @@ namespace winrt::Microsoft::Terminal::Control::implementation
                                                        keyDown) :
                                  true;
 
+        if (vkey && keyDown && _automationPeer)
+        {
+            get_self<TermControlAutomationPeer>(_automationPeer)->RecordKeyEvent(vkey);
+        }
+
         if (_cursorTimer)
         {
             // Manually show the cursor when a key is pressed. Restarting

src/cascadia/TerminalControl/TermControlAutomationPeer.cpp

@@ -28,6 +28,43 @@ namespace XamlAutomation
     using winrt::Windows::UI::Xaml::Automation::Provider::ITextRangeProvider;
 }
 
+static constexpr wchar_t UNICODE_NEWLINE{ L'\n' };
+
+// Method Description:
+// - creates a copy of the provided text with all of the control characters removed
+// Arguments:
+// - text: the string we're sanitizing
+// Return Value:
+// - a copy of "sanitized" with all of the control characters removed
+static std::wstring Sanitize(std::wstring_view text)
+{
+    std::wstring sanitized{ text };
+    sanitized.erase(std::remove_if(sanitized.begin(), sanitized.end(), [](wchar_t c) {
+                        return (c < UNICODE_SPACE && c != UNICODE_NEWLINE) || c == 0x7F /*DEL*/;
+                    }),
+                    sanitized.end());
+    return sanitized;
+}
+
+// Method Description:
+// - verifies if a given string has text that would be read by a screen reader.
+// - a string of control characters, for example, would not be read.
+// Arguments:
+// - text: the string we're validating
+// Return Value:
+// - true, if the text is readable. false, otherwise.
+static constexpr bool IsReadable(std::wstring_view text)
+{
+    for (const auto c : text)
+    {
+        if (c > UNICODE_SPACE)
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
 namespace winrt::Microsoft::Terminal::Control::implementation
 {
     TermControlAutomationPeer::TermControlAutomationPeer(TermControl* owner,
@@ -45,6 +82,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         _contentAutomationPeer.SelectionChanged([this](auto&&, auto&&) { SignalSelectionChanged(); });
         _contentAutomationPeer.TextChanged([this](auto&&, auto&&) { SignalTextChanged(); });
         _contentAutomationPeer.CursorChanged([this](auto&&, auto&&) { SignalCursorChanged(); });
+        _contentAutomationPeer.NewOutput([this](auto&&, hstring newOutput) { NotifyNewOutput(newOutput); });
         _contentAutomationPeer.ParentProvider(*this);
     };
 
@@ -68,6 +106,17 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         _contentAutomationPeer.SetControlPadding(padding);
     }
 
+    void TermControlAutomationPeer::RecordKeyEvent(const WORD vkey)
+    {
+        if (const auto charCode{ MapVirtualKey(vkey, MAPVK_VK_TO_CHAR) })
+        {
+            if (const auto keyEventChar{ gsl::narrow_cast<wchar_t>(charCode) }; IsReadable({ &keyEventChar, 1 }))
+            {
+                _keyEvents.emplace_back(keyEventChar);
+            }
+        }
+    }
+
     // Method Description:
     // - Signals the ui automation client that the terminal's selection has changed and should be updated
     // Arguments:
@@ -142,8 +191,66 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         });
     }
 
+    void TermControlAutomationPeer::NotifyNewOutput(std::wstring_view newOutput)
+    {
+        // Try to suppress any events (or event data)
+        // that is just the keypress the user made
+        auto sanitized{ Sanitize(newOutput) };
+        while (!_keyEvents.empty() && IsReadable(sanitized))
+        {
+            if (til::toupper_ascii(sanitized.front()) == _keyEvents.front())
+            {
+                // the key event's character (i.e. the "A" key) matches
+                // the output character (i.e. "a" or "A" text).
+                // We can assume that the output character resulted from
+                // the pressed key, so we can ignore it.
+                sanitized = sanitized.substr(1);
+                _keyEvents.pop_front();
+            }
+            else
+            {
+                // The output doesn't match,
+                // so clear the input stack and
+                // move on to fire the event.
+                _keyEvents.clear();
+                break;
+            }
+        }
+
+        // Suppress event if the remaining text is not readable
+        if (!IsReadable(sanitized))
+        {
+            return;
+        }
+
+        auto dispatcher{ Dispatcher() };
+        if (!dispatcher)
+        {
+            return;
+        }
+
+        // IMPORTANT:
+        // [1] make sure the scope returns a copy of "sanitized" so that it isn't accidentally deleted
+        // [2] AutomationNotificationProcessing::All --> ensures it can be interrupted by keyboard events
+        // [3] Do not "RunAsync(...).get()". For whatever reason, this causes NVDA to just not receive "SignalTextChanged()"'s events.
+        dispatcher.RunAsync(Windows::UI::Core::CoreDispatcherPriority::Normal, [weakThis{ get_weak() }, sanitizedCopy{ hstring{ sanitized } }]() {
+            if (auto strongThis{ weakThis.get() })
+            {
+                try
+                {
+                    strongThis->RaiseNotificationEvent(AutomationNotificationKind::ActionCompleted,
+                                                       AutomationNotificationProcessing::All,
+                                                       sanitizedCopy,
+                                                       L"TerminalTextOutput");
+                }
+                CATCH_LOG();
+            }
+        });
+    }
+
     hstring TermControlAutomationPeer::GetClassNameCore() const
     {
+        // IMPORTANT: Do NOT change the name. Screen readers like JAWS may be dependent on this being "TermControl".
         return L"TermControl";
     }
 

src/cascadia/TerminalControl/TermControlAutomationPeer.h

@@ -48,6 +48,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
 
         void UpdateControlBounds();
         void SetControlPadding(const Core::Padding padding);
+        void RecordKeyEvent(const WORD vkey);
 
 #pragma region FrameworkElementAutomationPeer
         hstring GetClassNameCore() const;
@@ -64,6 +65,7 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         void SignalSelectionChanged() override;
         void SignalTextChanged() override;
         void SignalCursorChanged() override;
+        void NotifyNewOutput(std::wstring_view newOutput) override;
 #pragma endregion
 
 #pragma region ITextProvider Pattern
@@ -78,5 +80,6 @@ namespace winrt::Microsoft::Terminal::Control::implementation
     private:
         winrt::Microsoft::Terminal::Control::implementation::TermControl* _termControl;
         Control::InteractivityAutomationPeer _contentAutomationPeer;
+        std::deque<wchar_t> _keyEvents;
     };
 }

src/cascadia/TerminalCore/Terminal.cpp

@@ -1002,6 +1002,11 @@ void Terminal::_WriteBuffer(const std::wstring_view& stringView)
         _AdjustCursorPosition(proposedCursorPosition);
     }
 
+    // Notify UIA of new text.
+    // It's important to do this here instead of in TextBuffer, because here you have access to the entire line of text,
+    // whereas TextBuffer writes it one character at a time via the OutputCellIterator.
+    _buffer->GetRenderTarget().TriggerNewTextNotification(stringView);
+
     cursor.EndDeferDrawing();
 }
 

src/cascadia/UnitTests_TerminalCore/ScrollTest.cpp

@@ -53,6 +53,7 @@ namespace
         };
         virtual void TriggerCircling(){};
         void TriggerTitleChange(){};
+        void TriggerNewTextNotification(const std::wstring_view){};
 
     private:
         std::optional<COORD> _triggerScrollDelta;

src/host/ScreenBufferRenderTarget.cpp

@@ -110,3 +110,13 @@ void ScreenBufferRenderTarget::TriggerTitleChange()
         pRenderer->TriggerTitleChange();
     }
 }
+
+void ScreenBufferRenderTarget::TriggerNewTextNotification(const std::wstring_view newText)
+{
+    auto* pRenderer = ServiceLocator::LocateGlobals().pRender;
+    const auto* pActive = &ServiceLocator::LocateGlobals().getConsoleInformation().GetActiveOutputBuffer().GetActiveBuffer();
+    if (pRenderer != nullptr && pActive == &_owner)
+    {
+        pRenderer->TriggerNewTextNotification(newText);
+    }
+}

src/host/ScreenBufferRenderTarget.hpp

@@ -39,6 +39,7 @@ class ScreenBufferRenderTarget final : public Microsoft::Console::Render::IRende
     void TriggerScroll(const COORD* const pcoordDelta) override;
     void TriggerCircling() override;
     void TriggerTitleChange() override;
+    void TriggerNewTextNotification(const std::wstring_view newText) override;
 
 private:
     SCREEN_INFORMATION& _owner;

src/renderer/atlas/AtlasEngine.api.cpp

@@ -141,6 +141,11 @@ constexpr HRESULT vec2_narrow(U x, U y, AtlasEngine::vec2<T>& out) noexcept
     return S_OK;
 }
 
+[[nodiscard]] HRESULT AtlasEngine::NotifyNewText(const std::wstring_view newText) noexcept
+{
+    return S_OK;
+}
+
 [[nodiscard]] HRESULT AtlasEngine::UpdateFont(const FontInfoDesired& fontInfoDesired, _Out_ FontInfo& fontInfo) noexcept
 {
     return UpdateFont(fontInfoDesired, fontInfo, {}, {});

src/renderer/atlas/AtlasEngine.h

@@ -35,6 +35,7 @@ namespace Microsoft::Console::Render
         [[nodiscard]] HRESULT InvalidateAll() noexcept override;
         [[nodiscard]] HRESULT InvalidateCircling(_Out_ bool* pForcePaint) noexcept override;
         [[nodiscard]] HRESULT InvalidateTitle(std::wstring_view proposedTitle) noexcept override;
+        [[nodiscard]] HRESULT NotifyNewText(const std::wstring_view newText) noexcept override;
         [[nodiscard]] HRESULT PrepareRenderInfo(const RenderFrameInfo& info) noexcept override;
         [[nodiscard]] HRESULT ResetLineTransform() noexcept override;
         [[nodiscard]] HRESULT PrepareLineTransform(LineRendition lineRendition, size_t targetRow, size_t viewportLeft) noexcept override;

src/renderer/base/RenderEngineBase.cpp

@@ -36,6 +36,11 @@ HRESULT RenderEngineBase::UpdateTitle(const std::wstring_view newTitle) noexcept
     return hr;
 }
 
+HRESULT RenderEngineBase::NotifyNewText(const std::wstring_view /*newText*/) noexcept
+{
+    return S_FALSE;
+}
+
 HRESULT RenderEngineBase::UpdateSoftFont(const gsl::span<const uint16_t> /*bitPattern*/,
                                          const SIZE /*cellSize*/,
                                          const size_t /*centeringHint*/) noexcept

src/renderer/base/renderer.cpp

@@ -493,6 +493,14 @@ void Renderer::TriggerTitleChange()
     NotifyPaintFrame();
 }
 
+void Renderer::TriggerNewTextNotification(const std::wstring_view newText)
+{
+    FOREACH_ENGINE(pEngine)
+    {
+        LOG_IF_FAILED(pEngine->NotifyNewText(newText));
+    }
+}
+
 // Routine Description:
 // - Update the title for a particular engine.
 // Arguments:

src/renderer/base/renderer.hpp

@@ -54,6 +54,8 @@ namespace Microsoft::Console::Render
         void TriggerCircling() override;
         void TriggerTitleChange() override;
 
+        void TriggerNewTextNotification(const std::wstring_view newText) override;
+
         void TriggerFontChange(const int iDpi,
                                const FontInfoDesired& FontInfoDesired,
                                _Out_ FontInfo& FontInfo);

src/renderer/inc/DummyRenderTarget.hpp

@@ -31,4 +31,5 @@ class DummyRenderTarget final : public Microsoft::Console::Render::IRenderTarget
     void TriggerScroll(const COORD* const /*pcoordDelta*/) override {}
     void TriggerCircling() override {}
     void TriggerTitleChange() override {}
+    void TriggerNewTextNotification(const std::wstring_view) override {}
 };

src/renderer/inc/IRenderEngine.hpp

@@ -69,6 +69,7 @@ namespace Microsoft::Console::Render
         [[nodiscard]] virtual HRESULT InvalidateAll() noexcept = 0;
         [[nodiscard]] virtual HRESULT InvalidateCircling(_Out_ bool* pForcePaint) noexcept = 0;
         [[nodiscard]] virtual HRESULT InvalidateTitle(std::wstring_view proposedTitle) noexcept = 0;
+        [[nodiscard]] virtual HRESULT NotifyNewText(const std::wstring_view newText) noexcept = 0;
         [[nodiscard]] virtual HRESULT PrepareRenderInfo(const RenderFrameInfo& info) noexcept = 0;
         [[nodiscard]] virtual HRESULT ResetLineTransform() noexcept = 0;
         [[nodiscard]] virtual HRESULT PrepareLineTransform(LineRendition lineRendition, size_t targetRow, size_t viewportLeft) noexcept = 0;

src/renderer/inc/IRenderTarget.hpp

@@ -45,6 +45,8 @@ namespace Microsoft::Console::Render
         virtual void TriggerScroll(const COORD* const pcoordDelta) = 0;
         virtual void TriggerCircling() = 0;
         virtual void TriggerTitleChange() = 0;
+
+        virtual void TriggerNewTextNotification(const std::wstring_view newText) = 0;
     };
 
     inline Microsoft::Console::Render::IRenderTarget::~IRenderTarget() {}

src/renderer/inc/RenderEngineBase.hpp

@@ -38,6 +38,8 @@ namespace Microsoft::Console::Render
 
         [[nodiscard]] HRESULT UpdateTitle(const std::wstring_view newTitle) noexcept override;
 
+        [[nodiscard]] HRESULT NotifyNewText(const std::wstring_view newText) noexcept override;
+
         [[nodiscard]] HRESULT UpdateSoftFont(const gsl::span<const uint16_t> bitPattern,
                                              const SIZE cellSize,
                                              const size_t centeringHint) noexcept override;