Skip to content

Commit fdcbab5

Browse files
Basic implementation of UIA navigation functions (#11412)
* commit in progress work for merge * Mostly working UIA tree nav functions * missed merge file * fix some merges * fix merge issue, add uia helpers * missed merge files * More merge fixesh * Change files * yarn format * address PR comments * missed a role mapping --------- Co-authored-by: Jon Thysell (JAUNTY) <[email protected]>
1 parent d5d19f5 commit fdcbab5

35 files changed

+633
-65
lines changed
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"type": "prerelease",
3+
"comment": "Add UIA Navigate to Fabric Providers",
4+
"packageName": "react-native-windows",
5+
"email": "[email protected]",
6+
"dependentChangeType": "patch"
7+
}

vnext/Microsoft.ReactNative/CompositionRootView.idl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ namespace Microsoft.ReactNative
7070

7171
Int64 SendMessage(UInt32 Msg, UInt64 WParam, Int64 LParam);
7272

73-
Object GetUiaProvider(UInt64 hwnd);
73+
Object GetUiaProvider();
7474

7575
//void OnPointerPressed(PointerPressedArgs args);
7676
//void OnMouseUp(Windows.Foundation.Point point);

vnext/Microsoft.ReactNative/Fabric/ComponentView.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#pragma once
66

77
#include <functional/functor.h>
8+
#include <inspectable.h>
89
#include <react/renderer/componentregistry/ComponentDescriptorProvider.h>
910
#include <react/renderer/components/view/TouchEventEmitter.h>
1011
#include <react/renderer/components/view/ViewProps.h>
@@ -61,6 +62,7 @@ struct IComponentView {
6162
virtual facebook::react::Tag tag() const noexcept = 0;
6263
virtual facebook::react::Tag hitTest(facebook::react::Point pt, facebook::react::Point &localPt) const noexcept = 0;
6364
virtual int64_t sendMessage(uint32_t msg, uint64_t wParam, int64_t lParam) noexcept = 0;
65+
virtual winrt::IInspectable EnsureUiaProvider() noexcept = 0;
6466
};
6567

6668
// Run fn on all nodes of the component view tree starting from this one until fn returns true

vnext/Microsoft.ReactNative/Fabric/Composition/AbiCompositionViewComponentView.cpp

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include "AbiCompositionViewComponentView.h"
88

99
#include <Fabric/DWriteHelpers.h>
10+
#include "CompositionDynamicAutomationProvider.h"
1011
#include "Unicode.h"
1112

1213
namespace Microsoft::ReactNative {
@@ -23,6 +24,14 @@ AbiCompositionViewComponentView::AbiCompositionViewComponentView(
2324
OuterVisual().InsertAt(m_visual, 0);
2425
}
2526

27+
std::shared_ptr<AbiCompositionViewComponentView> AbiCompositionViewComponentView::Create(
28+
const winrt::Microsoft::ReactNative::Composition::ICompositionContext &compContext,
29+
facebook::react::Tag tag,
30+
winrt::Microsoft::ReactNative::IReactViewComponentBuilder builder) noexcept {
31+
return std::shared_ptr<AbiCompositionViewComponentView>(
32+
new AbiCompositionViewComponentView(compContext, tag, builder));
33+
}
34+
2635
winrt::Microsoft::ReactNative::Composition::ReactCompositionViewComponentBuilder &
2736
AbiCompositionViewComponentView::Builder() noexcept {
2837
return *winrt::get_self<winrt::Microsoft::ReactNative::Composition::ReactCompositionViewComponentBuilder>(m_builder);
@@ -124,4 +133,12 @@ facebook::react::Tag AbiCompositionViewComponentView::hitTest(
124133
return -1;
125134
}
126135

136+
winrt::IInspectable AbiCompositionViewComponentView::EnsureUiaProvider() noexcept {
137+
if (m_uiaProvider == nullptr) {
138+
m_uiaProvider = winrt::make<winrt::Microsoft::ReactNative::implementation::CompositionDynamicAutomationProvider>(
139+
shared_from_this());
140+
}
141+
return m_uiaProvider;
142+
}
143+
127144
} // namespace Microsoft::ReactNative

vnext/Microsoft.ReactNative/Fabric/Composition/AbiCompositionViewComponentView.h

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,13 @@ namespace Microsoft::ReactNative {
1717

1818
struct AbiCompositionViewComponentView : CompositionBaseComponentView {
1919
using Super = CompositionBaseComponentView;
20-
AbiCompositionViewComponentView(
20+
21+
[[nodiscard]] static std::shared_ptr<AbiCompositionViewComponentView> Create(
2122
const winrt::Microsoft::ReactNative::Composition::ICompositionContext &compContext,
2223
facebook::react::Tag tag,
23-
winrt::Microsoft::ReactNative::IReactViewComponentBuilder builder);
24+
winrt::Microsoft::ReactNative::IReactViewComponentBuilder builder) noexcept;
25+
26+
winrt::IInspectable EnsureUiaProvider() noexcept override;
2427

2528
void mountChildComponentView(IComponentView &childComponentView, uint32_t index) noexcept override;
2629
void unmountChildComponentView(IComponentView &childComponentView, uint32_t index) noexcept override;
@@ -44,6 +47,11 @@ struct AbiCompositionViewComponentView : CompositionBaseComponentView {
4447
winrt::Microsoft::ReactNative::Composition::IVisual Visual() const noexcept override;
4548

4649
private:
50+
AbiCompositionViewComponentView(
51+
const winrt::Microsoft::ReactNative::Composition::ICompositionContext &compContext,
52+
facebook::react::Tag tag,
53+
winrt::Microsoft::ReactNative::IReactViewComponentBuilder builder);
54+
4755
winrt::Microsoft::ReactNative::Composition::ReactCompositionViewComponentBuilder &Builder() noexcept;
4856

4957
winrt::IInspectable m_handle;

vnext/Microsoft.ReactNative/Fabric/Composition/ComponentViewRegistry.cpp

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -46,30 +46,30 @@ ComponentViewDescriptor const &ComponentViewRegistry::dequeueComponentViewWithCo
4646
std::shared_ptr<CompositionBaseComponentView> view;
4747

4848
if (componentHandle == facebook::react::ViewShadowNode::Handle()) {
49-
view = std::make_shared<CompositionViewComponentView>(compContext, tag);
49+
view = CompositionViewComponentView::Create(compContext, tag);
5050
} else if (componentHandle == facebook::react::ParagraphShadowNode::Handle()) {
51-
view = std::make_shared<ParagraphComponentView>(compContext, tag);
51+
view = ParagraphComponentView::Create(compContext, tag);
5252
} else if (componentHandle == facebook::react::ScrollViewShadowNode::Handle()) {
53-
view = std::make_shared<ScrollViewComponentView>(compContext, tag);
53+
view = ScrollViewComponentView::Create(compContext, tag);
5454
} else if (componentHandle == facebook::react::ImageShadowNode::Handle()) {
55-
view = std::make_shared<ImageComponentView>(compContext, tag, m_context);
55+
view = ImageComponentView::Create(compContext, tag, m_context);
5656
} else if (componentHandle == facebook::react::WindowsTextInputShadowNode::Handle()) {
57-
view = std::make_shared<WindowsTextInputComponentView>(compContext, tag, m_context);
57+
view = WindowsTextInputComponentView::Create(compContext, tag, m_context);
5858
} else if (componentHandle == facebook::react::SwitchShadowNode::Handle()) {
59-
view = std::make_shared<SwitchComponentView>(compContext, tag, m_context);
59+
view = SwitchComponentView::Create(compContext, tag, m_context);
6060
} else if (componentHandle == facebook::react::RootShadowNode::Handle()) {
61-
view = std::make_shared<RootComponentView>(compContext, tag);
61+
view = RootComponentView::Create(compContext, tag);
6262
} else if (
6363
componentHandle == facebook::react::RawTextShadowNode::Handle() ||
6464
componentHandle == facebook::react::TextShadowNode::Handle()) {
6565
// Review - Why do we get asked for ComponentViews for Text/RawText... do these get used?
66-
view = std::make_shared<CompositionViewComponentView>(compContext, tag);
66+
view = CompositionViewComponentView::Create(compContext, tag);
6767
} else if (componentHandle == facebook::react::UnimplementedNativeViewShadowNode::Handle()) {
68-
view = std::make_shared<UnimplementedNativeViewComponentView>(compContext, tag);
68+
view = UnimplementedNativeViewComponentView::Create(compContext, tag);
6969
} else {
7070
auto descriptor =
7171
WindowsComponentDescriptorRegistry::FromProperties(m_context.Properties())->GetDescriptor(componentHandle);
72-
view = std::make_shared<AbiCompositionViewComponentView>(
72+
view = AbiCompositionViewComponentView::Create(
7373
compContext, tag, descriptor.as<winrt::Microsoft::ReactNative::IReactViewComponentBuilder>());
7474
}
7575

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
#include "pch.h"
2+
#include "CompositionDynamicAutomationProvider.h"
3+
#include <Fabric/ComponentView.h>
4+
#pragma warning(push)
5+
#pragma warning(disable : 4229)
6+
#define IN
7+
#define OUT
8+
#include <atlsafe.h>
9+
#pragma warning(pop)
10+
#include "RootComponentView.h"
11+
#include "UiaHelpers.h"
12+
13+
namespace winrt::Microsoft::ReactNative::implementation {
14+
15+
CompositionDynamicAutomationProvider::CompositionDynamicAutomationProvider(
16+
const std::shared_ptr<::Microsoft::ReactNative::CompositionBaseComponentView> &componentView) noexcept
17+
: m_view{std::static_pointer_cast<::Microsoft::ReactNative::IComponentView>(componentView)} {}
18+
19+
HRESULT __stdcall CompositionDynamicAutomationProvider::Navigate(
20+
NavigateDirection direction,
21+
IRawElementProviderFragment **pRetVal) {
22+
if (pRetVal == nullptr)
23+
return E_POINTER;
24+
25+
return UiaNavigateHelper(m_view, direction, *pRetVal);
26+
}
27+
28+
// Implementations should return NULL for a top-level element that is hosted in a window. Other elements should return
29+
// an array that contains UiaAppendRuntimeId (defined in Uiautomationcoreapi.h), followed by a value that is unique
30+
// within an instance of the fragment.
31+
//
32+
// We'll use the react tag as our identifier for those situations
33+
HRESULT __stdcall CompositionDynamicAutomationProvider::GetRuntimeId(SAFEARRAY **pRetVal) {
34+
if (pRetVal == nullptr)
35+
return E_POINTER;
36+
37+
*pRetVal = nullptr;
38+
39+
auto strongView = m_view.view();
40+
41+
if (!strongView)
42+
return UIA_E_ELEMENTNOTAVAILABLE;
43+
44+
CComSafeArray<int32_t> runtimeId;
45+
auto hr = runtimeId.Create(2);
46+
47+
if (FAILED(hr))
48+
return hr;
49+
50+
runtimeId[0] = UiaAppendRuntimeId;
51+
runtimeId[1] = strongView->tag();
52+
53+
*pRetVal = runtimeId.Detach();
54+
55+
return S_OK;
56+
}
57+
58+
HRESULT __stdcall CompositionDynamicAutomationProvider::get_BoundingRectangle(UiaRect *pRetVal) {
59+
if (pRetVal == nullptr)
60+
return E_POINTER;
61+
62+
return S_OK;
63+
}
64+
65+
HRESULT __stdcall CompositionDynamicAutomationProvider::GetEmbeddedFragmentRoots(SAFEARRAY **pRetVal) {
66+
if (pRetVal == nullptr)
67+
return E_POINTER;
68+
69+
*pRetVal = nullptr;
70+
71+
return S_OK;
72+
}
73+
74+
HRESULT __stdcall CompositionDynamicAutomationProvider::SetFocus(void) {
75+
return S_OK;
76+
}
77+
78+
HRESULT __stdcall CompositionDynamicAutomationProvider::get_FragmentRoot(IRawElementProviderFragmentRoot **pRetVal) {
79+
if (pRetVal == nullptr)
80+
return E_POINTER;
81+
82+
auto strongView = m_view.view();
83+
84+
if (!strongView)
85+
return UIA_E_ELEMENTNOTAVAILABLE;
86+
87+
auto rootCV = strongView->rootComponentView();
88+
if (rootCV == nullptr)
89+
return UIA_E_ELEMENTNOTAVAILABLE;
90+
91+
auto uiaProvider = rootCV->EnsureUiaProvider();
92+
if (uiaProvider != nullptr) {
93+
winrt::com_ptr<IRawElementProviderFragmentRoot> spReps;
94+
uiaProvider.as(spReps);
95+
*pRetVal = spReps.detach();
96+
}
97+
98+
return S_OK;
99+
}
100+
101+
HRESULT __stdcall CompositionDynamicAutomationProvider::get_ProviderOptions(ProviderOptions *pRetVal) {
102+
if (pRetVal == nullptr)
103+
return E_POINTER;
104+
105+
*pRetVal = ProviderOptions_ServerSideProvider | ProviderOptions_UseComThreading;
106+
return S_OK;
107+
}
108+
109+
HRESULT __stdcall CompositionDynamicAutomationProvider::GetPatternProvider(PATTERNID patternId, IUnknown **pRetVal) {
110+
if (pRetVal == nullptr)
111+
return E_POINTER;
112+
113+
*pRetVal = nullptr;
114+
115+
return S_OK;
116+
}
117+
118+
long GetControlType(const std::string &role) noexcept {
119+
if (role == "adjustable") {
120+
return UIA_SliderControlTypeId;
121+
} else if (role == "group" || role == "search" || role == "radiogroup" || role == "timer" || role.empty()) {
122+
return UIA_GroupControlTypeId;
123+
} else if (role == "button" || role == "imagebutton" || role == "switch" || role == "togglebutton") {
124+
return UIA_ButtonControlTypeId;
125+
} else if (role == "checkbox") {
126+
return UIA_CheckBoxControlTypeId;
127+
} else if (role == "combobox") {
128+
return UIA_ComboBoxControlTypeId;
129+
} else if (role == "alert" || role == "header" || role == "summary" || role == "text") {
130+
return UIA_TextControlTypeId;
131+
} else if (role == "image") {
132+
return UIA_ImageControlTypeId;
133+
} else if (role == "keyboardkey") {
134+
return UIA_CustomControlTypeId;
135+
} else if (role == "link") {
136+
return UIA_HyperlinkControlTypeId;
137+
}
138+
// list and listitem were added by RNW to better support UIA Control Types
139+
else if (role == "list") {
140+
return UIA_ListControlTypeId;
141+
} else if (role == "listitem") {
142+
return UIA_ListItemControlTypeId;
143+
} else if (role == "menu") {
144+
return UIA_MenuControlTypeId;
145+
} else if (role == "menubar") {
146+
return UIA_MenuBarControlTypeId;
147+
} else if (role == "menuitem") {
148+
return UIA_MenuItemControlTypeId;
149+
}
150+
// If role is "none", remove the element from the control tree
151+
// and expose it as a plain element would in the raw tree.
152+
else if (role == "none") {
153+
return UIA_GroupControlTypeId;
154+
} else if (role == "progressbar") {
155+
return UIA_ProgressBarControlTypeId;
156+
} else if (role == "radio") {
157+
return UIA_RadioButtonControlTypeId;
158+
} else if (role == "scrollbar") {
159+
return UIA_ScrollBarControlTypeId;
160+
} else if (role == "spinbutton") {
161+
return UIA_SpinnerControlTypeId;
162+
} else if (role == "splitbutton") {
163+
return UIA_SplitButtonControlTypeId;
164+
} else if (role == "tab") {
165+
return UIA_TabItemControlTypeId;
166+
} else if (role == "tablist") {
167+
return UIA_TabControlTypeId;
168+
} else if (role == "toolbar") {
169+
return UIA_ToolBarControlTypeId;
170+
} else if (role == "tree") {
171+
return UIA_TreeControlTypeId;
172+
} else if (role == "treeitem") {
173+
return UIA_TreeItemControlTypeId;
174+
}
175+
assert(false);
176+
return UIA_GroupControlTypeId;
177+
}
178+
179+
HRESULT __stdcall CompositionDynamicAutomationProvider::GetPropertyValue(PROPERTYID propertyId, VARIANT *pRetVal) {
180+
if (pRetVal == nullptr)
181+
return E_POINTER;
182+
183+
auto strongView = m_view.view();
184+
if (strongView == nullptr)
185+
return UIA_E_ELEMENTNOTAVAILABLE;
186+
187+
auto props = std::static_pointer_cast<const facebook::react::ViewProps>(strongView->props());
188+
if (props == nullptr)
189+
return UIA_E_ELEMENTNOTAVAILABLE;
190+
191+
switch (propertyId) {
192+
case UIA_ControlTypePropertyId: {
193+
pRetVal->vt = VT_I4;
194+
auto role = props->accessibilityRole;
195+
pRetVal->lVal = GetControlType(role);
196+
break;
197+
}
198+
case UIA_AutomationIdPropertyId: {
199+
pRetVal->vt = VT_BSTR;
200+
auto testId = props->testId;
201+
CComBSTR temp(testId.c_str());
202+
pRetVal->bstrVal = temp.Detach();
203+
break;
204+
}
205+
case UIA_NamePropertyId: {
206+
pRetVal->vt = VT_BSTR;
207+
auto name = props->accessibilityLabel;
208+
CComBSTR temp(name.c_str());
209+
pRetVal->bstrVal = temp.Detach();
210+
break;
211+
}
212+
}
213+
214+
return S_OK;
215+
}
216+
217+
HRESULT __stdcall CompositionDynamicAutomationProvider::get_HostRawElementProvider(
218+
IRawElementProviderSimple **pRetVal) {
219+
if (pRetVal == nullptr)
220+
return E_POINTER;
221+
222+
*pRetVal = nullptr;
223+
224+
return S_OK;
225+
}
226+
227+
} // namespace winrt::Microsoft::ReactNative::implementation

0 commit comments

Comments
 (0)