Implement getBoundingRect and ElementProviderFromPoint for Fabric

change/react-native-windows-dd7d3409-cce5-4725-b7d9-722b0e961a78.json

@@ -0,0 +1,7 @@
+{
+  "type": "prerelease",
+  "comment": "working impl of getBoundingRect and ElementProviderFromPoint",
+  "packageName": "react-native-windows",
+  "email": "[email protected]",
+  "dependentChangeType": "patch"
+}

vnext/Microsoft.ReactNative/Fabric/ComponentView.h

@@ -60,7 +60,12 @@ struct IComponentView {
   virtual facebook::react::SharedTouchEventEmitter touchEventEmitterAtPoint(facebook::react::Point pt) noexcept = 0;
   virtual facebook::react::SharedTouchEventEmitter touchEventEmitter() noexcept = 0;
   virtual facebook::react::Tag tag() const noexcept = 0;
-  virtual facebook::react::Tag hitTest(facebook::react::Point pt, facebook::react::Point &localPt) const noexcept = 0;
+  // By default, hitTests according the pointerEvents prop on the Component.
+  // If ignorePointerEvents = true, all Components are treated as valid targets
+  virtual facebook::react::Tag hitTest(
+      facebook::react::Point pt,
+      facebook::react::Point &localPt,
+      bool ignorePointerEvents = false) const noexcept = 0;
   virtual int64_t sendMessage(uint32_t msg, uint64_t wParam, int64_t lParam) noexcept = 0;
   virtual winrt::IInspectable EnsureUiaProvider() noexcept = 0;
 };

vnext/Microsoft.ReactNative/Fabric/Composition/AbiCompositionViewComponentView.cpp

@@ -119,10 +119,11 @@ winrt::Microsoft::ReactNative::Composition::IVisual AbiCompositionViewComponentV
 
 facebook::react::Tag AbiCompositionViewComponentView::hitTest(
     facebook::react::Point pt,
-    facebook::react::Point &localPt) const noexcept {
+    facebook::react::Point &localPt,
+    bool ignorePointerEvents) const noexcept {
   facebook::react::Point ptLocal{pt.x - m_layoutMetrics.frame.origin.x, pt.y - m_layoutMetrics.frame.origin.y};
 
-  if ((m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
+  if ((ignorePointerEvents || m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
        m_props->pointerEvents == facebook::react::PointerEventsMode::BoxOnly) &&
       ptLocal.x >= 0 && ptLocal.x <= m_layoutMetrics.frame.size.width && ptLocal.y >= 0 &&
       ptLocal.y <= m_layoutMetrics.frame.size.height) {

vnext/Microsoft.ReactNative/Fabric/Composition/AbiCompositionViewComponentView.h

@@ -43,7 +43,8 @@ struct AbiCompositionViewComponentView : CompositionBaseComponentView {
   std::vector<facebook::react::ComponentDescriptorProvider> supplementalComponentDescriptorProviders() noexcept
       override;
   facebook::react::Props::Shared props() noexcept override;
-  facebook::react::Tag hitTest(facebook::react::Point pt, facebook::react::Point &localPt) const noexcept override;
+  facebook::react::Tag hitTest(facebook::react::Point pt, facebook::react::Point &localPt, bool ignorePointerEvents)
+      const noexcept override;
   winrt::Microsoft::ReactNative::Composition::IVisual Visual() const noexcept override;
 
  private:

vnext/Microsoft.ReactNative/Fabric/Composition/ComponentViewRegistry.cpp

@@ -58,7 +58,7 @@ ComponentViewDescriptor const &ComponentViewRegistry::dequeueComponentViewWithCo
   } else if (componentHandle == facebook::react::SwitchShadowNode::Handle()) {
     view = SwitchComponentView::Create(compContext, tag, m_context);
   } else if (componentHandle == facebook::react::RootShadowNode::Handle()) {
-    view = RootComponentView::Create(compContext, tag);
+    view = RootComponentView::Create(compContext, tag, m_context);
   } else if (
       componentHandle == facebook::react::RawTextShadowNode::Handle() ||
       componentHandle == facebook::react::TextShadowNode::Handle()) {

vnext/Microsoft.ReactNative/Fabric/Composition/CompositionDynamicAutomationProvider.cpp

@@ -59,6 +59,30 @@ HRESULT __stdcall CompositionDynamicAutomationProvider::get_BoundingRectangle(Ui
   if (pRetVal == nullptr)
     return E_POINTER;
 
+  auto hr = UiaGetBoundingRectangleHelper(m_view, *pRetVal);
+  if (FAILED(hr))
+    return hr;
+
+  // Since get_BoundingRectangle needs to provide real screen coordinates back to the UIA client
+  // we'll use the FragmentRoot's origin to offset our rect because that should have been taken
+  // into account already.
+  winrt::com_ptr<IRawElementProviderFragmentRoot> spFragmentRoot = nullptr;
+  hr = get_FragmentRoot(spFragmentRoot.put());
+  if (FAILED(hr))
+    return hr;
+
+  auto spFragment = spFragmentRoot.try_as<IRawElementProviderFragment>();
+  if (spFragment == nullptr)
+    return E_FAIL;
+
+  UiaRect rect;
+  hr = spFragment->get_BoundingRectangle(&rect);
+  if (FAILED(hr))
+    return hr;
+
+  pRetVal->left += rect.left;
+  pRetVal->top += rect.top;
+
   return S_OK;
 }
 
@@ -89,10 +113,9 @@ HRESULT __stdcall CompositionDynamicAutomationProvider::get_FragmentRoot(IRawEle
     return UIA_E_ELEMENTNOTAVAILABLE;
 
   auto uiaProvider = rootCV->EnsureUiaProvider();
-  if (uiaProvider != nullptr) {
-    winrt::com_ptr<IRawElementProviderFragmentRoot> spReps;
-    uiaProvider.as(spReps);
-    *pRetVal = spReps.detach();
+  auto spFragmentRoot = uiaProvider.try_as<IRawElementProviderFragmentRoot>();
+  if (spFragmentRoot) {
+    *pRetVal = spFragmentRoot.detach();
   }
 
   return S_OK;

vnext/Microsoft.ReactNative/Fabric/Composition/CompositionRootAutomationProvider.cpp

@@ -55,6 +55,7 @@ HRESULT __stdcall CompositionRootAutomationProvider::get_HostRawElementProvider(
   if (pRetVal == nullptr)
     return E_POINTER;
 
+  // TODO: assumes windowed
   if (!IsWindow(m_hwnd))
     return UIA_E_ELEMENTNOTAVAILABLE;
 
@@ -67,6 +68,24 @@ HRESULT __stdcall CompositionRootAutomationProvider::get_BoundingRectangle(UiaRe
   if (pRetVal == nullptr)
     return E_POINTER;
 
+  // TODO: Need host site offsets
+  // Assume we're hosted in some other visual-based hosting site
+  if (m_hwnd == nullptr || !IsWindow(m_hwnd)) {
+    return UiaGetBoundingRectangleHelper(m_view, *pRetVal);
+  }
+
+  POINT point{0, 0};
+  ClientToScreen(m_hwnd, &point);
+  pRetVal->left = point.x;
+  pRetVal->top = point.y;
+  RECT rect;
+  GetClientRect(m_hwnd, &rect);
+  point.x = rect.right;
+  point.y = rect.bottom;
+  ClientToScreen(m_hwnd, &point);
+  pRetVal->width = point.x - pRetVal->left;
+  pRetVal->height = point.y - pRetVal->top;
+
   return S_OK;
 }
 
@@ -97,6 +116,31 @@ HRESULT __stdcall CompositionRootAutomationProvider::ElementProviderFromPoint(
 
   *pRetVal = nullptr;
 
+  auto strongView = m_view.view();
+
+  if (strongView == nullptr) {
+    return UIA_E_ELEMENTNOTAVAILABLE;
+  }
+
+  auto spRootView = strongView->rootComponentView();
+  if (spRootView == nullptr) {
+    return UIA_E_ELEMENTNOTAVAILABLE;
+  }
+
+  if (m_hwnd == nullptr || !IsWindow(m_hwnd)) {
+    // TODO: Add support for non-HWND based hosting
+    return E_FAIL;
+  }
+
+  POINT clientPoint{static_cast<LONG>(x), static_cast<LONG>(y)};
+  ScreenToClient(m_hwnd, &clientPoint);
+
+  auto provider = spRootView->UiaProviderFromPoint(clientPoint);
+  auto spFragment = provider.try_as<IRawElementProviderFragment>();
+  if (spFragment) {
+    *pRetVal = spFragment.detach();
+  }
+
   return S_OK;
 }
 
@@ -117,9 +161,10 @@ HRESULT __stdcall CompositionRootAutomationProvider::GetFocus(IRawElementProvide
 
   if (focusedComponent) {
     auto focusedUiaProvider = focusedComponent->EnsureUiaProvider();
-    winrt::com_ptr<IRawElementProviderFragment> spFragment;
-    focusedUiaProvider.as(spFragment);
-    *pRetVal = spFragment.detach();
+    auto spFragment = focusedUiaProvider.try_as<IRawElementProviderFragment>();
+    if (spFragment) {
+      *pRetVal = spFragment.detach();
+    }
   }
 
   return S_OK;

vnext/Microsoft.ReactNative/Fabric/Composition/CompositionViewComponentView.cpp

@@ -1267,21 +1267,23 @@ void CompositionViewComponentView::updateProps(
   m_props = std::static_pointer_cast<facebook::react::ViewProps const>(props);
 }
 
-facebook::react::Tag CompositionViewComponentView::hitTest(facebook::react::Point pt, facebook::react::Point &localPt)
-    const noexcept {
+facebook::react::Tag CompositionViewComponentView::hitTest(
+    facebook::react::Point pt,
+    facebook::react::Point &localPt,
+    bool ignorePointerEvents) const noexcept {
   facebook::react::Point ptLocal{pt.x - m_layoutMetrics.frame.origin.x, pt.y - m_layoutMetrics.frame.origin.y};
 
   facebook::react::Tag targetTag;
 
-  if ((m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
+  if ((ignorePointerEvents || m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
        m_props->pointerEvents == facebook::react::PointerEventsMode::BoxNone) &&
       std::any_of(m_children.rbegin(), m_children.rend(), [&targetTag, &ptLocal, &localPt](auto child) {
         targetTag = static_cast<const CompositionBaseComponentView *>(child)->hitTest(ptLocal, localPt);
         return targetTag != -1;
       }))
     return targetTag;
 
-  if ((m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
+  if ((ignorePointerEvents || m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
        m_props->pointerEvents == facebook::react::PointerEventsMode::BoxOnly) &&
       ptLocal.x >= 0 && ptLocal.x <= m_layoutMetrics.frame.size.width && ptLocal.y >= 0 &&
       ptLocal.y <= m_layoutMetrics.frame.size.height) {

vnext/Microsoft.ReactNative/Fabric/Composition/CompositionViewComponentView.h

@@ -114,7 +114,10 @@ struct CompositionViewComponentView : public CompositionBaseComponentView {
 
   facebook::react::Props::Shared props() noexcept override;
 
-  facebook::react::Tag hitTest(facebook::react::Point pt, facebook::react::Point &localPt) const noexcept override;
+  facebook::react::Tag hitTest(
+      facebook::react::Point pt,
+      facebook::react::Point &localPt,
+      bool ignorePointerEvents = false) const noexcept override;
   bool ScrollWheel(facebook::react::Point pt, int32_t delta) noexcept override;
 
   winrt::Microsoft::ReactNative::Composition::IVisual Visual() const noexcept override;

vnext/Microsoft.ReactNative/Fabric/Composition/ImageComponentView.cpp

@@ -362,21 +362,23 @@ facebook::react::Props::Shared ImageComponentView::props() noexcept {
   return m_props;
 }
 
-facebook::react::Tag ImageComponentView::hitTest(facebook::react::Point pt, facebook::react::Point &localPt)
-    const noexcept {
+facebook::react::Tag ImageComponentView::hitTest(
+    facebook::react::Point pt,
+    facebook::react::Point &localPt,
+    bool ignorePointerEvents) const noexcept {
   facebook::react::Point ptLocal{pt.x - m_layoutMetrics.frame.origin.x, pt.y - m_layoutMetrics.frame.origin.y};
 
   facebook::react::Tag targetTag;
 
-  if ((m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
+  if ((ignorePointerEvents || m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
        m_props->pointerEvents == facebook::react::PointerEventsMode::BoxNone) &&
       std::any_of(m_children.rbegin(), m_children.rend(), [&targetTag, &ptLocal, &localPt](auto child) {
         targetTag = static_cast<const CompositionBaseComponentView *>(child)->hitTest(ptLocal, localPt);
         return targetTag != -1;
       }))
     return targetTag;
 
-  if ((m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
+  if ((ignorePointerEvents || m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
        m_props->pointerEvents == facebook::react::PointerEventsMode::BoxOnly) &&
       ptLocal.x >= 0 && ptLocal.x <= m_layoutMetrics.frame.size.width && ptLocal.y >= 0 &&
       ptLocal.y <= m_layoutMetrics.frame.size.height) {

vnext/Microsoft.ReactNative/Fabric/Composition/ImageComponentView.h

@@ -49,7 +49,8 @@ struct ImageComponentView : CompositionBaseComponentView {
   facebook::react::Props::Shared props() noexcept override;
   void OnRenderingDeviceLost() noexcept override;
 
-  facebook::react::Tag hitTest(facebook::react::Point pt, facebook::react::Point &localPt) const noexcept override;
+  facebook::react::Tag hitTest(facebook::react::Point pt, facebook::react::Point &localPt, bool ignorePointerEvents)
+      const noexcept override;
   winrt::Microsoft::ReactNative::Composition::IVisual Visual() const noexcept override;
   bool focusable() const noexcept override;
 

vnext/Microsoft.ReactNative/Fabric/Composition/ParagraphComponentView.cpp

@@ -99,8 +99,10 @@ facebook::react::Props::Shared ParagraphComponentView::props() noexcept {
   return m_props;
 }
 
-facebook::react::Tag ParagraphComponentView::hitTest(facebook::react::Point pt, facebook::react::Point &localPt)
-    const noexcept {
+facebook::react::Tag ParagraphComponentView::hitTest(
+    facebook::react::Point pt,
+    facebook::react::Point &localPt,
+    bool ignorePointerEvents) const noexcept {
   facebook::react::Point ptLocal{pt.x - m_layoutMetrics.frame.origin.x, pt.y - m_layoutMetrics.frame.origin.y};
 
   facebook::react::Tag targetTag;
@@ -115,7 +117,7 @@ facebook::react::Tag ParagraphComponentView::hitTest(facebook::react::Point pt,
       return targetTag;
       */
 
-  if ((m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
+  if ((ignorePointerEvents || m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
        m_props->pointerEvents == facebook::react::PointerEventsMode::BoxOnly) &&
       ptLocal.x >= 0 && ptLocal.x <= m_layoutMetrics.frame.size.width && ptLocal.y >= 0 &&
       ptLocal.y <= m_layoutMetrics.frame.size.height) {

vnext/Microsoft.ReactNative/Fabric/Composition/ParagraphComponentView.h

@@ -36,7 +36,8 @@ struct ParagraphComponentView : CompositionBaseComponentView {
   void finalizeUpdates(RNComponentViewUpdateMask updateMask) noexcept override;
   void prepareForRecycle() noexcept override;
   facebook::react::Props::Shared props() noexcept override;
-  facebook::react::Tag hitTest(facebook::react::Point pt, facebook::react::Point &localPt) const noexcept override;
+  facebook::react::Tag hitTest(facebook::react::Point pt, facebook::react::Point &localPt, bool ignorePointerEvents)
+      const noexcept override;
   void OnRenderingDeviceLost() noexcept override;
   facebook::react::SharedTouchEventEmitter touchEventEmitterAtPoint(facebook::react::Point pt) noexcept override;
 

vnext/Microsoft.ReactNative/Fabric/Composition/RootComponentView.cpp

@@ -6,20 +6,23 @@
 
 #include "RootComponentView.h"
 
+#include <Fabric/FabricUIManagerModule.h>
 #include "CompositionRootAutomationProvider.h"
 #include "CompositionRootView.h"
 
 namespace Microsoft::ReactNative {
 
 RootComponentView::RootComponentView(
     const winrt::Microsoft::ReactNative::Composition::ICompositionContext &compContext,
-    facebook::react::Tag tag)
-    : Super(compContext, tag) {}
+    facebook::react::Tag tag,
+    winrt::Microsoft::ReactNative::ReactContext const &reactContext)
+    : Super(compContext, tag), m_context(reactContext) {}
 
 std::shared_ptr<RootComponentView> RootComponentView::Create(
     const winrt::Microsoft::ReactNative::Composition::ICompositionContext &compContext,
-    facebook::react::Tag tag) noexcept {
-  return std::shared_ptr<RootComponentView>(new RootComponentView(compContext, tag));
+    facebook::react::Tag tag,
+    winrt::Microsoft::ReactNative::ReactContext const &reactContext) noexcept {
+  return std::shared_ptr<RootComponentView>(new RootComponentView(compContext, tag, reactContext));
 }
 
 RootComponentView *RootComponentView::rootComponentView() noexcept {
@@ -113,4 +116,23 @@ std::shared_ptr<RootComponentView> RootComponentView::getPtr() {
   return std::static_pointer_cast<RootComponentView>(shared_from_this());
 }
 
+winrt::IInspectable RootComponentView::UiaProviderFromPoint(const POINT &ptPixels) noexcept {
+  facebook::react::Point ptDips{
+      static_cast<facebook::react::Float>(ptPixels.x) / m_layoutMetrics.pointScaleFactor,
+      static_cast<facebook::react::Float>(ptPixels.y) / m_layoutMetrics.pointScaleFactor};
+
+  facebook::react::Point localPt;
+  auto tag = hitTest(ptDips, localPt, true);
+
+  auto uiManager = ::Microsoft::ReactNative::FabricUIManager::FromProperties(m_context.Properties());
+  if (uiManager == nullptr)
+    return nullptr;
+
+  auto view = uiManager->GetViewRegistry().findComponentViewWithTag(tag);
+  if (view == nullptr)
+    return nullptr;
+
+  return view->EnsureUiaProvider();
+}
+
 } // namespace Microsoft::ReactNative

vnext/Microsoft.ReactNative/Fabric/Composition/RootComponentView.h

@@ -18,7 +18,8 @@ struct RootComponentView : CompositionViewComponentView {
 
   [[nodiscard]] static std::shared_ptr<RootComponentView> Create(
       const winrt::Microsoft::ReactNative::Composition::ICompositionContext &compContext,
-      facebook::react::Tag tag) noexcept;
+      facebook::react::Tag tag,
+      winrt::Microsoft::ReactNative::ReactContext const &reactContext) noexcept;
 
   std::shared_ptr<RootComponentView> getPtr();
 
@@ -33,11 +34,15 @@ struct RootComponentView : CompositionViewComponentView {
 
   winrt::IInspectable EnsureUiaProvider() noexcept override;
 
+  winrt::IInspectable UiaProviderFromPoint(const POINT &ptPixels) noexcept;
+
  private:
   RootComponentView(
       const winrt::Microsoft::ReactNative::Composition::ICompositionContext &compContext,
-      facebook::react::Tag tag);
+      facebook::react::Tag tag,
+      winrt::Microsoft::ReactNative::ReactContext const &reactContext);
   ::Microsoft::ReactNative::IComponentView *m_focusedComponent = nullptr;
+  winrt::Microsoft::ReactNative::ReactContext m_context;
 };
 
 } // namespace Microsoft::ReactNative

vnext/Microsoft.ReactNative/Fabric/Composition/ScrollViewComponentView.cpp

@@ -335,24 +335,26 @@ void ScrollViewComponentView::ensureVisual() noexcept {
   }
 }
 
-facebook::react::Tag ScrollViewComponentView::hitTest(facebook::react::Point pt, facebook::react::Point &localPt)
-    const noexcept {
+facebook::react::Tag ScrollViewComponentView::hitTest(
+    facebook::react::Point pt,
+    facebook::react::Point &localPt,
+    bool ignorePointerEvents) const noexcept {
   facebook::react::Point ptViewport{pt.x - m_layoutMetrics.frame.origin.x, pt.y - m_layoutMetrics.frame.origin.y};
 
   facebook::react::Point ptContent{
       ptViewport.x + m_scrollVisual.ScrollPosition().x / m_layoutMetrics.pointScaleFactor,
       ptViewport.y + m_scrollVisual.ScrollPosition().y / m_layoutMetrics.pointScaleFactor};
 
   facebook::react::Tag targetTag;
-  if ((m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
+  if ((ignorePointerEvents || m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
        m_props->pointerEvents == facebook::react::PointerEventsMode::BoxNone) &&
       std::any_of(m_children.rbegin(), m_children.rend(), [&targetTag, &ptContent, &localPt](auto child) {
         targetTag = static_cast<const CompositionBaseComponentView *>(child)->hitTest(ptContent, localPt);
         return targetTag != -1;
       }))
     return targetTag;
 
-  if ((m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
+  if ((ignorePointerEvents || m_props->pointerEvents == facebook::react::PointerEventsMode::Auto ||
        m_props->pointerEvents == facebook::react::PointerEventsMode::BoxOnly) &&
       ptViewport.x >= 0 && ptViewport.x <= m_layoutMetrics.frame.size.width && ptViewport.y >= 0 &&
       ptViewport.y <= m_layoutMetrics.frame.size.height) {