From 341de6adc4da4ea8bba55a7f40ef97bac4f2d278 Mon Sep 17 00:00:00 2001 From: zjxi Date: Tue, 7 Jan 2025 15:02:36 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix:=E6=97=A0=E9=9A=9C=E7=A2=8D=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=A4=9Axcomponent=E5=9C=BA=E6=99=AF=EF=BC=8C?= =?UTF-8?q?=E4=BD=86=E4=BC=9A=E5=81=B6=E7=8E=B0=E9=87=8E=E6=8C=87=E9=92=88?= =?UTF-8?q?=E9=97=AA=E9=80=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zjxi --- .../ohos_accessibility_bridge.cpp | 53 +++++++++++++++++-- .../accessibility/ohos_accessibility_bridge.h | 5 ++ .../src/main/cpp/types/libflutter/index.d.ets | 4 +- .../main/ets/embedding/ohos/FlutterPage.ets | 30 +++++++++-- shell/platform/ohos/library_loader.cpp | 3 ++ .../ohos/napi/platform_view_ohos_napi.cpp | 19 +++++++ .../ohos/napi/platform_view_ohos_napi.h | 4 ++ .../platform/ohos/ohos_xcomponent_adapter.cpp | 23 +++++--- shell/platform/ohos/ohos_xcomponent_adapter.h | 6 +-- 9 files changed, 127 insertions(+), 20 deletions(-) diff --git a/shell/platform/ohos/accessibility/ohos_accessibility_bridge.cpp b/shell/platform/ohos/accessibility/ohos_accessibility_bridge.cpp index dbb1a076e9..09bbe8cece 100644 --- a/shell/platform/ohos/accessibility/ohos_accessibility_bridge.cpp +++ b/shell/platform/ohos/accessibility/ohos_accessibility_bridge.cpp @@ -31,6 +31,37 @@ const int32_t OhosAccessibilityBridge::OHOS_API_VERSION = OH_GetSdkApiVersion(); std::unique_ptr OhosAccessibilityBridge::bridgeInstance_ = nullptr; +/** + * flutter无障碍相关语义树、句柄指针provider等参数 + * 跟随xcomponentid显示切换，而加载对应xcomponent的语义树和provider指针 + * @NOTE: 当屏幕同时显示多个xcomponent时，无法通过聚焦事件而触发xcomponentid改变 + */ +void OhosAccessibilityBridge::AccessibiltiyChangesWithXComponentId() +{ + auto xcompMap = XComponentAdapter::GetInstance()->xcomponetMap_; + // 获取当前显示的xcomponetid + std::string currXcompId = XComponentAdapter::GetInstance()->currentXComponentId_; + if (xcomponentId_ == currXcompId) { return; } + + auto it = xcompMap.find(currXcompId); + if (!xcompMap.empty() && it != xcompMap.end()) { + // 更新xcompid，shellholderId，provider指针 + xcomponentId_ = currXcompId; + native_shell_holder_id_ = std::stoll(it->second->shellholderId_); + provider_ = it->second->accessibilityProvider_; + g_flutterSemanticsTree = g_flutterSemanticsTreeXComponents[xcomponentId_]; + g_parentChildIdVec = g_parentChildIdVecXComponents[xcomponentId_]; + + FML_DLOG(INFO) << "AccessibiltiyChangesWithXComponentId -> xcomponentid:" << xcomponentId_; + } else { + xcomponentId_ = "oh_flutter_1"; + provider_ = nullptr; + g_flutterSemanticsTree = g_flutterSemanticsTreeXComponents[xcomponentId_]; + g_parentChildIdVec = g_parentChildIdVecXComponents[xcomponentId_]; + FML_DLOG(INFO) << "AccessibiltiyChangesWithXComponentId -> xcomponentid:" << xcomponentId_; + } +} + /** * 采用局部静态变量配合call-once特性，实现线程安全的单例模式 */ @@ -54,7 +85,8 @@ void OhosAccessibilityBridge::OnOhosAccessibilityStateChange( bool ohosAccessibilityEnabled, int64_t shellholderId) { native_shell_holder_id_ = shellholderId; - provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; + AccessibiltiyChangesWithXComponentId(); + // provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; nativeAccessibilityChannel_ = std::make_shared(); accessibilityFeatures_ = std::make_shared(); @@ -81,7 +113,7 @@ void OhosAccessibilityBridge::UpdateSemantics( flutter::CustomAccessibilityActionUpdates actions) { FML_DLOG(INFO) << "OhosAccessibilityBridge::UpdateSemantics()"; - provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; + // provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; std::vector updatedFlutterNodes; // 当flutter页面状态更新（路由新页面）时，自动请求root节点组件获焦（规避滑动组件更新干扰） @@ -124,6 +156,10 @@ void OhosAccessibilityBridge::UpdateSemantics( } } + // 将更新后的flutter语义树和父子节点id映射缓存，保存到相应的xcomponent里面 + g_flutterSemanticsTreeXComponents[xcomponentId_] = g_flutterSemanticsTree; + g_parentChildIdVecXComponents[xcomponentId_] = g_parentChildIdVec; + // 页面内容更新事件 Flutter_SendAccessibilityAsyncEvent(0, ArkUI_AccessibilityEventType:: @@ -270,7 +306,8 @@ void OhosAccessibilityBridge::FlutterScrollExecution( void OhosAccessibilityBridge::RequestFocusWhenPageUpdate(int32_t requestFocusId) { if (OHOS_API_VERSION < 13) { return; } - provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; + AccessibiltiyChangesWithXComponentId(); + // provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; CHECK_NULL_PTR_RET_VOID(provider_, RequestFocusWhenPageUpdate); auto OH_ArkUI_CreateAccessibilityEventInfo = @@ -978,6 +1015,8 @@ int32_t OhosAccessibilityBridge::FindAccessibilityNodeInfosById( << elementId << " mode=" << mode; CHECK_NULL_PTR_WITH_RET(elementList, FindAccessibilityNodeInfosById); + AccessibiltiyChangesWithXComponentId(); + if (g_flutterSemanticsTree.size() == 0) { FML_DLOG(INFO) << "FindAccessibilityNodeInfosById g_flutterSemanticsTree is null"; @@ -1370,6 +1409,8 @@ int32_t OhosAccessibilityBridge::ExecuteAccessibilityAction( << elementId << " action=" << action; CHECK_NULL_PTR_WITH_RET(actionArguments, ExecuteAccessibilityAction); + AccessibiltiyChangesWithXComponentId(); + // 获取当前elementid对应的flutter语义节点 auto flutterNode = GetFlutterSemanticsNode(static_cast(elementId)); if (!g_flutterSemanticsTree.count(flutterNode.id)) { @@ -1565,7 +1606,8 @@ void OhosAccessibilityBridge::Flutter_SendAccessibilityAnnounceEvent( ArkUI_AccessibilityEventType eventType) { if (OHOS_API_VERSION < 13) { return; } - provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; + AccessibiltiyChangesWithXComponentId(); + // provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; CHECK_NULL_PTR_RET_VOID(provider_, Flutter_SendAccessibilityAnnounceEvent); // 创建并设置屏幕朗读事件 @@ -1614,7 +1656,8 @@ void OhosAccessibilityBridge::Flutter_SendAccessibilityAsyncEvent( ArkUI_AccessibilityEventType eventType) { if (OHOS_API_VERSION < 13) { return; } - provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; + AccessibiltiyChangesWithXComponentId(); + // provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; CHECK_NULL_PTR_RET_VOID(provider_, Flutter_SendAccessibilityAsyncEvent); // 创建eventInfo对象 diff --git a/shell/platform/ohos/accessibility/ohos_accessibility_bridge.h b/shell/platform/ohos/accessibility/ohos_accessibility_bridge.h index f4e03364ce..d202f7e6ec 100644 --- a/shell/platform/ohos/accessibility/ohos_accessibility_bridge.h +++ b/shell/platform/ohos/accessibility/ohos_accessibility_bridge.h @@ -78,6 +78,7 @@ public: OhosAccessibilityBridge(const OhosAccessibilityBridge&) = delete; OhosAccessibilityBridge& operator=(const OhosAccessibilityBridge&) = delete; + std::string xcomponentId_; bool isFlutterNavigated_; int64_t native_shell_holder_id_; ArkUI_AccessibilityProvider* provider_; @@ -161,6 +162,10 @@ private: std::shared_ptr nativeAccessibilityChannel_; std::shared_ptr accessibilityFeatures_; + std::unordered_map> g_flutterSemanticsTreeXComponents; + std::unordered_map>> g_parentChildIdVecXComponents; + std::unordered_map> g_screenRectMapXComponents; + std::unordered_map g_flutterSemanticsTree; std::vector> g_parentChildIdVec; std::unordered_map g_screenRectMap; diff --git a/shell/platform/ohos/flutter_embedding/flutter/src/main/cpp/types/libflutter/index.d.ets b/shell/platform/ohos/flutter_embedding/flutter/src/main/cpp/types/libflutter/index.d.ets index 279c97baa1..a683639641 100644 --- a/shell/platform/ohos/flutter_embedding/flutter/src/main/cpp/types/libflutter/index.d.ets +++ b/shell/platform/ohos/flutter_embedding/flutter/src/main/cpp/types/libflutter/index.d.ets @@ -175,4 +175,6 @@ export const nativeUnicodeIsEmojiModifierBase: (code: number) => number; export const nativeUnicodeIsVariationSelector: (code: number) => number; -export const nativeUnicodeIsRegionalIndicatorSymbol: (code: number) => number; \ No newline at end of file +export const nativeUnicodeIsRegionalIndicatorSymbol: (code: number) => number; + +export const nativeGetXComponentId: (id: string) => number; \ No newline at end of file diff --git a/shell/platform/ohos/flutter_embedding/flutter/src/main/ets/embedding/ohos/FlutterPage.ets b/shell/platform/ohos/flutter_embedding/flutter/src/main/ets/embedding/ohos/FlutterPage.ets index 533e1e6a51..64497b3d8c 100644 --- a/shell/platform/ohos/flutter_embedding/flutter/src/main/ets/embedding/ohos/FlutterPage.ets +++ b/shell/platform/ohos/flutter_embedding/flutter/src/main/ets/embedding/ohos/FlutterPage.ets @@ -17,7 +17,7 @@ import Log from '../../util/Log'; import { FlutterView } from '../../view/FlutterView'; import FlutterManager from './FlutterManager'; import { DVModel, DVModelChildren, DynamicView } from '../../view/DynamicView/dynamicView'; - +import flutter from 'libflutter.so'; const TAG = "FlutterPage"; export const OHOS_FLUTTER_PAGE_UPDATE = "ohos_flutter_page_update"; @@ -56,7 +56,19 @@ export struct FlutterPage { .onLoad((context) => { this.flutterView?.onSurfaceCreated() Log.d(TAG, "XComponent onLoad "); - this.flutterView?.onAccessibilityIsOpen() + // 当xcomponent窗口部分显示或完全隐藏时触发回调 + this.getUIContext()?.getAttachedFrameNodeById(this.viewId)?.commonEvent.setOnVisibleAreaApproximateChange( + { ratios: [0.0, 1.0], expectedUpdateInterval: 0 }, + (ratioInc: boolean, ratio: number) => { + if (ratioInc) { + Log.i(TAG, "setOnVisibleAreaApproximateChange -> xcomponentId: " + this.viewId + + " ratioInc: " + ratioInc + " ratio: " + ratio); + flutter.nativeGetXComponentId(this.viewId); + // 保证获取xcomponentid之后再使用无障碍 + this.flutterView?.onAccessibilityIsOpen(); + } + } + ) }) .onDestroy(() => { Log.d(TAG, "XComponent onDestroy "); @@ -136,7 +148,19 @@ export struct FlutterPage { .onLoad((context) => { this.flutterView?.onSurfaceCreated() Log.d(TAG, "XComponent onLoad "); - this.flutterView?.onAccessibilityIsOpen() + // 当xcomponent窗口部分显示或完全隐藏时触发回调 + this.getUIContext()?.getAttachedFrameNodeById(this.viewId)?.commonEvent.setOnVisibleAreaApproximateChange( + { ratios: [0.0, 1.0], expectedUpdateInterval: 0 }, + (ratioInc: boolean, ratio: number) => { + if (ratioInc) { + Log.i(TAG, "setOnVisibleAreaApproximateChange -> xcomponentId: " + this.viewId + + " ratioInc: " + ratioInc + " ratio: " + ratio); + flutter.nativeGetXComponentId(this.viewId); + // 保证获取xcomponentid之后再使用无障碍 + this.flutterView?.onAccessibilityIsOpen(); + } + } + ) }) .onDestroy(() => { Log.d(TAG, "XComponent onDestroy "); diff --git a/shell/platform/ohos/library_loader.cpp b/shell/platform/ohos/library_loader.cpp index 7b2c7da1c0..91ceec8956 100644 --- a/shell/platform/ohos/library_loader.cpp +++ b/shell/platform/ohos/library_loader.cpp @@ -190,6 +190,9 @@ static napi_value Init(napi_env env, napi_value exports) { DECLARE_NAPI_FUNCTION( "nativeUnicodeIsRegionalIndicatorSymbol", flutter::PlatformViewOHOSNapi::nativeUnicodeIsRegionalIndicatorSymbol), + DECLARE_NAPI_FUNCTION( + "nativeGetXComponentId", + flutter::PlatformViewOHOSNapi::nativeGetXComponentId), }; FML_DLOG(INFO) << "Init NAPI size=" << sizeof(desc) / sizeof(desc[0]); diff --git a/shell/platform/ohos/napi/platform_view_ohos_napi.cpp b/shell/platform/ohos/napi/platform_view_ohos_napi.cpp index d1bf9ed472..78f4cc842f 100644 --- a/shell/platform/ohos/napi/platform_view_ohos_napi.cpp +++ b/shell/platform/ohos/napi/platform_view_ohos_napi.cpp @@ -2277,4 +2277,23 @@ napi_value PlatformViewOHOSNapi::nativeUnicodeIsRegionalIndicatorSymbol(napi_env napi_create_int32(env, (int)is_emoji, &result); return result; } + +napi_value PlatformViewOHOSNapi::nativeGetXComponentId(napi_env env, napi_callback_info info) +{ + size_t argc = 1; + napi_value args[1] = {nullptr}; + napi_get_cb_info(env, info, &argc, args, nullptr, nullptr); + + std::string xcomponentId; + bool ret = fml::napi::GetString(env, args[0], xcomponentId); + if (ret != napi_ok) { + FML_DLOG(ERROR) << "nativeGetXComponentId xcomponentId: " << xcomponentId; + return nullptr; + } + // obtain the current visible xcomponent id from the ets callback event + XComponentAdapter::GetInstance()->currentXComponentId_ = xcomponentId; + FML_DLOG(ERROR) << "nativeGetXComponentId -> xcomponentId: " << xcomponentId; + return nullptr; +} + } // namespace flutter \ No newline at end of file diff --git a/shell/platform/ohos/napi/platform_view_ohos_napi.h b/shell/platform/ohos/napi/platform_view_ohos_napi.h index 013557aeae..84ea45af1e 100644 --- a/shell/platform/ohos/napi/platform_view_ohos_napi.h +++ b/shell/platform/ohos/napi/platform_view_ohos_napi.h @@ -269,6 +269,10 @@ class PlatformViewOHOSNapi { napi_env env, napi_callback_info info); + static napi_value nativeGetXComponentId( + napi_env env, + napi_callback_info info); + private: static napi_env env_; napi_ref ref_napi_obj_; diff --git a/shell/platform/ohos/ohos_xcomponent_adapter.cpp b/shell/platform/ohos/ohos_xcomponent_adapter.cpp index 424123a0c6..c130995549 100644 --- a/shell/platform/ohos/ohos_xcomponent_adapter.cpp +++ b/shell/platform/ohos/ohos_xcomponent_adapter.cpp @@ -99,6 +99,10 @@ void XComponentAdapter::AttachFlutterEngine(std::string& id, auto findIter = xcomponetMap_.find(id); if (findIter != xcomponetMap_.end()) { findIter->second->AttachFlutterEngine(shellholderId); + // register the OH_ArkUI accessibility callbacks + if (OH_GetSdkApiVersion() >= 13) { + findIter->second->RegisterArkUIAccessibilityService(findIter->second->nativeXComponent_, shellholderId); + } } } @@ -201,6 +205,8 @@ void OnSurfaceDestroyedCB(OH_NativeXComponent* component, void* window) { if(it->second->nativeXComponent_ == component) { it->second->OnSurfaceDestroyed(component, window); delete it->second; + // 将当前要销毁的xcomponent对应的无障碍provider指针置nullptr + it->second->accessibilityProvider_ = nullptr; it = XComponentAdapter::GetInstance()->xcomponetMap_.erase(it); } else { ++it; @@ -364,7 +370,8 @@ void XComponentBase::DetachFlutterEngine() { isEngineAttached_ = false; } -void XComponentBase::RegisterArkUIAccessibilityService(OH_NativeXComponent* nativeXComponent) +void XComponentBase::RegisterArkUIAccessibilityService( + OH_NativeXComponent* nativeXComponent, const std::string& shellholderId) { BindAccessibilityProviderCallback(); @@ -385,8 +392,11 @@ void XComponentBase::RegisterArkUIAccessibilityService(OH_NativeXComponent* nati OH_ArkUI_AccessibilityProviderRegisterCallback(a11yProvider, &accessibilityProviderCallback_) ); - XComponentAdapter::GetInstance()->accessibilityProvider_ = a11yProvider; - + auto* base = XComponentAdapter::GetInstance()->xcomponetMap_[id_]; + base->shellholderId_ = shellholderId; + base->accessibilityProvider_ = a11yProvider; + base->nativeXComponent_ = nativeXComponent; + FML_DLOG(INFO) << "RegisterArkUIAccessibilityService is finished"; } @@ -396,10 +406,6 @@ void XComponentBase::SetNativeXComponent(OH_NativeXComponent* nativeXComponent){ BindXComponentCallback(); OH_NativeXComponent_RegisterCallback(nativeXComponent_, &callback_); OH_NativeXComponent_RegisterMouseEventCallback(nativeXComponent_, &mouseCallback_); - // register the OH_ArkUI accessibility callbacks - if (OH_GetSdkApiVersion() >= 13) { - RegisterArkUIAccessibilityService(nativeXComponent_); - } } } @@ -456,7 +462,8 @@ void XComponentBase::OnSurfaceChanged(OH_NativeXComponent* component, void* wind void XComponentBase::OnSurfaceDestroyed(OH_NativeXComponent* component, void* window) { window_ = nullptr; - XComponentAdapter::GetInstance()->accessibilityProvider_ = nullptr; + // XComponentAdapter::GetInstance()->accessibilityProvider_ = nullptr; + LOGD("XComponentManger::OnSurfaceDestroyed"); if (isEngineAttached_) { PlatformViewOHOSNapi::SurfaceDestroyed(std::stoll(shellholderId_)); diff --git a/shell/platform/ohos/ohos_xcomponent_adapter.h b/shell/platform/ohos/ohos_xcomponent_adapter.h index 7698e3a8f2..3500fabdb4 100644 --- a/shell/platform/ohos/ohos_xcomponent_adapter.h +++ b/shell/platform/ohos/ohos_xcomponent_adapter.h @@ -51,7 +51,8 @@ public: void OnDispatchMouseEvent(OH_NativeXComponent* component, void* window); void OnDispatchMouseWheelEvent(mouseWheelEvent event); - void RegisterArkUIAccessibilityService(OH_NativeXComponent* nativeXComponent); + void RegisterArkUIAccessibilityService( + OH_NativeXComponent* nativeXComponent, const std::string& shellholderId); OH_NativeXComponent_TouchEvent touchEvent_; OH_NativeXComponent_Callback callback_; @@ -68,7 +69,6 @@ public: uint64_t height_; OhosTouchProcessor ohosTouchProcessor_; ArkUI_AccessibilityProvider* accessibilityProvider_; - }; class XComponentAdapter { @@ -85,7 +85,7 @@ class XComponentAdapter { public: std::map xcomponetMap_; - ArkUI_AccessibilityProvider* accessibilityProvider_; + std::string currentXComponentId_; private: static XComponentAdapter mXComponentAdapter; -- Gitee From 2a72ee9798de354d21dbe3d2389c35096517e996 Mon Sep 17 00:00:00 2001 From: zjxi Date: Mon, 20 Jan 2025 13:02:02 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8D=E6=97=A0=E9=9A=9C?= =?UTF-8?q?=E7=A2=8D=E5=8D=95=E5=AE=9E=E4=BE=8Bxcomponent=E5=88=87?= =?UTF-8?q?=E6=8D=A2=E9=97=AA=E9=80=80=E9=97=AE=E9=A2=98=E3=80=81=E7=9B=B8?= =?UTF-8?q?=E5=AF=B9-=E7=BB=9D=E5=AF=B9=E5=9D=90=E6=A0=87=E8=BD=AC?= =?UTF-8?q?=E6=8D=A2=E7=AE=97=E6=B3=95bug=E5=AF=BC=E8=87=B4hypium=E4=B8=8B?= =?UTF-8?q?=E6=BB=91=E5=8A=A8=E5=B8=83=E5=B1=80=E8=8A=82=E7=82=B9=E6=98=BE?= =?UTF-8?q?=E7=A4=BA=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zjxi --- .../ohos_accessibility_bridge.cpp | 197 +++++++++--------- .../accessibility/ohos_accessibility_bridge.h | 20 +- .../platform/ohos/ohos_xcomponent_adapter.cpp | 25 ++- shell/platform/ohos/ohos_xcomponent_adapter.h | 7 +- 4 files changed, 139 insertions(+), 110 deletions(-) diff --git a/shell/platform/ohos/accessibility/ohos_accessibility_bridge.cpp b/shell/platform/ohos/accessibility/ohos_accessibility_bridge.cpp index 09bbe8cece..16f9d4b7b6 100644 --- a/shell/platform/ohos/accessibility/ohos_accessibility_bridge.cpp +++ b/shell/platform/ohos/accessibility/ohos_accessibility_bridge.cpp @@ -21,8 +21,6 @@ #include "flutter/shell/common/platform_view.h" #include "flutter/shell/platform/embedder/embedder.h" #include "flutter/shell/platform/ohos/ohos_shell_holder.h" -#include "third_party/skia/include/core/SkMatrix.h" -#include "third_party/skia/include/core/SkScalar.h" #include "flutter/shell/platform/ohos/ohos_xcomponent_adapter.h" namespace flutter { @@ -48,14 +46,11 @@ void OhosAccessibilityBridge::AccessibiltiyChangesWithXComponentId() // 更新xcompid，shellholderId，provider指针 xcomponentId_ = currXcompId; native_shell_holder_id_ = std::stoll(it->second->shellholderId_); - provider_ = it->second->accessibilityProvider_; g_flutterSemanticsTree = g_flutterSemanticsTreeXComponents[xcomponentId_]; g_parentChildIdVec = g_parentChildIdVecXComponents[xcomponentId_]; - FML_DLOG(INFO) << "AccessibiltiyChangesWithXComponentId -> xcomponentid:" << xcomponentId_; } else { xcomponentId_ = "oh_flutter_1"; - provider_ = nullptr; g_flutterSemanticsTree = g_flutterSemanticsTreeXComponents[xcomponentId_]; g_parentChildIdVec = g_parentChildIdVecXComponents[xcomponentId_]; FML_DLOG(INFO) << "AccessibiltiyChangesWithXComponentId -> xcomponentid:" << xcomponentId_; @@ -75,7 +70,7 @@ OhosAccessibilityBridge* OhosAccessibilityBridge::GetInstance() } OhosAccessibilityBridge::OhosAccessibilityBridge() - : isFlutterNavigated_(false), provider_(nullptr), + : isFlutterNavigated_(false), isAccessibilityEnabled_(false) {} /** @@ -86,7 +81,6 @@ void OhosAccessibilityBridge::OnOhosAccessibilityStateChange( { native_shell_holder_id_ = shellholderId; AccessibiltiyChangesWithXComponentId(); - // provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; nativeAccessibilityChannel_ = std::make_shared(); accessibilityFeatures_ = std::make_shared(); @@ -99,9 +93,18 @@ void OhosAccessibilityBridge::OnOhosAccessibilityStateChange( } } -void OhosAccessibilityBridge::SetNativeShellHolderId(int64_t id) +/** + * build the id mapping betwween parent node and its children nodes + */ +void OhosAccessibilityBridge::BuildParentChildNodeIdRelation( + const SemanticsNodeExtent& node) { - this->native_shell_holder_id_ = id; + if (!IsNodeVisible(node)) { return; } + for (const auto& childId : node.childrenInTraversalOrder) { + auto childNode = GetFlutterSemanticsNode(childId); + if (!IsNodeVisible(childNode)) { continue; } + g_parentChildIdVec.emplace_back(std::make_pair(node.id, childId)); + } } /** @@ -113,14 +116,10 @@ void OhosAccessibilityBridge::UpdateSemantics( flutter::CustomAccessibilityActionUpdates actions) { FML_DLOG(INFO) << "OhosAccessibilityBridge::UpdateSemantics()"; - // provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; std::vector updatedFlutterNodes; // 当flutter页面状态更新（路由新页面）时，自动请求root节点组件获焦（规避滑动组件更新干扰） if (isFlutterNavigated_) { - Flutter_SendAccessibilityAsyncEvent(0, - ArkUI_AccessibilityEventType:: - ARKUI_ACCESSIBILITY_NATIVE_EVENT_TYPE_PAGE_STATE_UPDATE); RequestFocusWhenPageUpdate(0); isFlutterNavigated_ = false; } @@ -129,16 +128,20 @@ void OhosAccessibilityBridge::UpdateSemantics( for (const auto& item : update) { // 获取当前更新的节点node const auto& node = item.second; - + FML_DLOG(INFO) << "*#*#* node.id=" << node.id; // 更新扩展的SemanticsNode信息 auto nodeEx = UpdatetSemanticsNodeExtent(node); + // 构建flutter无障碍语义节点树 + g_flutterSemanticsTree[nodeEx.id] = nodeEx; + // 构建flutter节点的父子id映射关系 + BuildParentChildNodeIdRelation(nodeEx); + //print semantics node and flags info for debugging GetSemanticsNodeDebugInfo(nodeEx); GetSemanticsFlagsDebugInfo(nodeEx); - // 构建flutter无障碍语义节点树 - g_flutterSemanticsTree[nodeEx.id] = nodeEx; + if (!IsNodeVisible(nodeEx)) { continue; } // 若当前节点为获焦 if (IsNodeFocused(nodeEx)) { @@ -147,12 +150,7 @@ void OhosAccessibilityBridge::UpdateSemantics( // 若当前节点和更新前节点信息不同，则加入更新节点数组 if (nodeEx.hadPreviousConfig) { updatedFlutterNodes.emplace_back(nodeEx); - } - - // 获取当前flutter节点的全部子节点数量，并构建父子节点id映射关系 - int32_t childNodeCount = nodeEx.childrenInTraversalOrder.size(); - for (int32_t i = 0; i < childNodeCount; i++) { - g_parentChildIdVec.emplace_back(std::make_pair(nodeEx.id, nodeEx.childrenInTraversalOrder[i])); + FML_DLOG(INFO) << "updatedFlutterNodes -> node.id=" << nodeEx.id; } } @@ -167,10 +165,12 @@ void OhosAccessibilityBridge::UpdateSemantics( LOGD("Flutter_SendAccessibilityAsyncEvent -> PAGE_CONTENT_UPDATE"); /* 针对更新后的节点进行事件处理 */ - for (const auto& nodeEx: updatedFlutterNodes) { + for (auto& nodeEx: updatedFlutterNodes) { + FML_DLOG(INFO) << "*#*#* updated node.id=" << nodeEx.id; + // 当滑动节点产生滑动，并执行滑动处理 if (HasScrolled(nodeEx)) { - LOGD("UpdateSemantics -> has scrolled"); + LOGD("UpdateSemantics -> nodeId = %{public}d has scrolled", nodeEx.id); auto OH_ArkUI_CreateAccessibilityElementInfo = OhosAccessibilityDDL::DLLoadCreateElemInfoFunc(ArkUIAccessibilityConstant::ARKUI_CREATE_NODE); CHECK_DLL_NULL_PTR(OH_ArkUI_CreateAccessibilityElementInfo); @@ -192,16 +192,16 @@ void OhosAccessibilityBridge::UpdateSemantics( _elementInfo = nullptr; } - // 判断是否触发liveRegion活动区，当前节点是否活跃 + // 判断是否触发liveRegion活动区，当前节点是否活跃 nodeEx.HasFlag(FLAGS_::kIsLiveRegion) if (nodeEx.HasFlag(FLAGS_::kIsLiveRegion) && HasChangedLabel(nodeEx)) { - FML_DLOG(INFO) << "UpdateSemantics -> page content update, nodeEx.id=" << nodeEx.id; + FML_DLOG(INFO) << "liveRegion -> page content update, nodeEx.id=" << nodeEx.id; Flutter_SendAccessibilityAsyncEvent(static_cast(nodeEx.id), ArkUI_AccessibilityEventType:: ARKUI_ACCESSIBILITY_NATIVE_EVENT_TYPE_PAGE_CONTENT_UPDATE); - LOGD("Flutter_SendAccessibilityAsyncEvent -> PAGE_CONTENT_UPDATE"); } } - + // calculate the global tranfomr matrix for each node + ComputeGlobalTransform(); // 输出flutter语义树相关重要语义信息debug日志 GetSemanticsDebugInfo(); FML_DLOG(INFO) << "=== UpdateSemantics() is finished ==="; @@ -241,7 +241,7 @@ void OhosAccessibilityBridge::FlutterScrollExecution( if (node.HasAction(ACTIONS_::kScrollUp) || node.HasAction(ACTIONS_::kScrollDown)) { } else if (node.HasAction(ACTIONS_::kScrollLeft) || - node.HasAction(ACTIONS_::kScrollRight)) { + node.HasAction(ACTIONS_::kScrollRight)) { LOGD("current flutterNode has scroll up/down/left/right"); } @@ -306,8 +306,8 @@ void OhosAccessibilityBridge::FlutterScrollExecution( void OhosAccessibilityBridge::RequestFocusWhenPageUpdate(int32_t requestFocusId) { if (OHOS_API_VERSION < 13) { return; } - AccessibiltiyChangesWithXComponentId(); - // provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; + std::lock_guard lock(XComponentAdapter::GetInstance()->mutex_); + auto provider_ = XComponentAdapter::GetInstance()->GetAccessibilityProvider(xcomponentId_); CHECK_NULL_PTR_RET_VOID(provider_, RequestFocusWhenPageUpdate); auto OH_ArkUI_CreateAccessibilityEventInfo = @@ -499,68 +499,66 @@ std::pair OhosAccessibilityBridge::GetRealScaleFactor() } /** - * flutter语义树中相对坐标系转化为屏幕绝对坐标的映射算法实现 - * NOTE:目前暂未考虑旋转、透视场景，不影响屏幕朗读功能 - * (SkMatrix::kMSkewX, SkMatrix::kMSkewY, SkMatrix::kMPersp0, - * SkMatrix::kMPersp1, SkMatrix::kMPersp2) + * calculate the global transform matrix for each node */ -void OhosAccessibilityBridge::FlutterRelativeRectToScreenRect( - SemanticsNodeExtent currNode) -{ - // 获取当前flutter节点的相对rect - auto [currLeft, currTop, currRight, currBottom] = currNode.rect; - - // 获取当前flutter节点的缩放、平移、透视等矩阵坐标转换以下矩阵坐标变换参数 - SkMatrix transform = currNode.transform.asM33(); - auto _kMScaleX = transform.get(SkMatrix::kMScaleX); - auto _kMTransX = transform.get(SkMatrix::kMTransX); - auto _kMScaleY = transform.get(SkMatrix::kMScaleY); - auto _kMTransY = transform.get(SkMatrix::kMTransY); - - // 获取当前flutter节点的父节点的相对rect - int32_t parentId = GetParentId(currNode.id); - auto parentNode = GetFlutterSemanticsNode(parentId); - if (g_flutterSemanticsTree.find(parentId) == g_flutterSemanticsTree.end()) { - LOGE("FlutterRelativeRectToScreenRect: GetFlutterSemanticsNode id=%{public}d null", parentId); - } - - // 获取当前flutter节点的父节点的绝对坐标 - auto [realParentLeft, realParentTop, realParentRight, realParentBottom] = GetAbsoluteScreenRect(parentNode); - - //获取root节点的绝对坐标 - auto rootNode = GetFlutterSemanticsNode(0); - auto rootRect = GetAbsoluteScreenRect(rootNode); - auto rootWidth = rootRect.right; - auto rootHeight = rootRect.bottom; - - // 获取真实缩放系数 - auto [realScaleXFactor, realScaleYFactor] = GetRealScaleFactor(); - - // 更新后的节点屏幕坐标，子节点的屏幕绝对坐标转换，包括offset偏移值计算、缩放系数变换 - bool isRectScaleChanged = _kMScaleX > 1 && _kMScaleY > 1; - float newLeft = isRectScaleChanged ? - currLeft + _kMTransX * _kMScaleX : (currLeft + _kMTransX) * realScaleXFactor + realParentLeft; - float newTop = isRectScaleChanged ? - currTop + _kMTransY * _kMScaleY : (currTop + _kMTransY) * realScaleYFactor + realParentTop; - float newRight = isRectScaleChanged ? - currRight * _kMScaleX : newLeft + currRight * realScaleXFactor; - float newBottom = isRectScaleChanged ? - currBottom * _kMScaleY : newTop + currBottom * realScaleYFactor; - - // 若子节点rect超过父节点则跳过显示（单个屏幕显示不下，滑动再重新显示） - const bool isExceedScreeArea = newLeft < realParentLeft || newTop < realParentTop || - newRight > realParentRight || newBottom > realParentBottom || - newLeft >= newRight || newTop >= newBottom || - newRight > rootWidth || newBottom > rootHeight; - if (isExceedScreeArea) { - FML_DLOG(WARNING) << "RelativeRectToScreenRect -> childRect exceeds parentRect {Id: " - << currNode.id << ", (" << newLeft << ", " << newTop - << ", " << newRight << ", " << newBottom << ")}"; - // 防止滑动场景下绿框坐标超出屏幕范围，进行正则化处理 - SetAbsoluteScreenRect(currNode, rootWidth, rootHeight, 0, 0); - } else { - SetAbsoluteScreenRect(currNode, newLeft, newTop, newRight, newBottom); +void OhosAccessibilityBridge::ComputeGlobalTransform() +{ + std::queue semanticsQue; + + auto root = GetFlutterSemanticsNode(0); + semanticsQue.push(root); + g_globalTransformMap[root.id] = root.transform; + + while (!semanticsQue.empty()) { + uint32_t queSize = semanticsQue.size(); + for (uint32_t i=0; i Transformed points can't make a rect "; + } + globalRect.setBounds(points, 4); + + SetAbsoluteScreenRect(node, globalRect.left(), globalRect.top(), + globalRect.right(), globalRect.bottom()); } /** @@ -692,7 +690,7 @@ void OhosAccessibilityBridge::FlutterSetElementInfoProperties( rect = {static_cast(left), static_cast(top), static_cast(right), static_cast(bottom)}; } else { // 若当前节点为id >= 1的节点 - FlutterRelativeRectToScreenRect(flutterNode); + RelativeRectToScreenRect(flutterNode); auto [left, top, right, bottom] = GetAbsoluteScreenRect(flutterNode); rect = {static_cast(left), static_cast(top), static_cast(right), static_cast(bottom)}; @@ -926,7 +924,8 @@ void OhosAccessibilityBridge::FlutterSetElementInfoProperties( OhosAccessibilityDDL::DLLoadSetElemStringFunc(ArkUIAccessibilityConstant::ARKUI_SET_A11Y_LEVEL); CHECK_DLL_NULL_PTR(OH_ArkUI_AccessibilityElementInfoSetAccessibilityLevel); ARKUI_ACCESSIBILITY_CALL_CHECK( - OH_ArkUI_AccessibilityElementInfoSetAccessibilityLevel(elementInfoFromList, "yes"); + OH_ArkUI_AccessibilityElementInfoSetAccessibilityLevel( + elementInfoFromList, componentTypeName != OTHER_WIDGET_NAME ? "yes" : "no"); ); // 无障碍组，设置为true时表示该组件及其所有子组件为一整个可以选中的组件，无障碍服务将不再关注其子组件内容。默认值：false auto OH_ArkUI_AccessibilityElementInfoSetAccessibilityGroup = @@ -954,6 +953,7 @@ std::vector OhosAccessibilityBridge::GetLevelOrderTraversalTree(int32_t uint32_t queSize = semanticsQue.size(); for (uint32_t i=0; i(currNode.id)); @@ -961,6 +961,7 @@ std::vector OhosAccessibilityBridge::GetLevelOrderTraversalTree(int32_t currNode.childrenInTraversalOrder.end()); for (const auto& childId: currNode.childrenInTraversalOrder) { auto childNode = GetFlutterSemanticsNode(childId); + semanticsQue.push(childNode); } } @@ -1607,7 +1608,8 @@ void OhosAccessibilityBridge::Flutter_SendAccessibilityAnnounceEvent( { if (OHOS_API_VERSION < 13) { return; } AccessibiltiyChangesWithXComponentId(); - // provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; + std::lock_guard lock(XComponentAdapter::GetInstance()->mutex_); + auto provider_ = XComponentAdapter::GetInstance()->GetAccessibilityProvider(xcomponentId_); CHECK_NULL_PTR_RET_VOID(provider_, Flutter_SendAccessibilityAnnounceEvent); // 创建并设置屏幕朗读事件 @@ -1657,7 +1659,8 @@ void OhosAccessibilityBridge::Flutter_SendAccessibilityAsyncEvent( { if (OHOS_API_VERSION < 13) { return; } AccessibiltiyChangesWithXComponentId(); - // provider_ = XComponentAdapter::GetInstance()->accessibilityProvider_; + std::lock_guard lock(XComponentAdapter::GetInstance()->mutex_); + auto provider_ = XComponentAdapter::GetInstance()->GetAccessibilityProvider(xcomponentId_); CHECK_NULL_PTR_RET_VOID(provider_, Flutter_SendAccessibilityAsyncEvent); // 创建eventInfo对象 @@ -2010,6 +2013,10 @@ void OhosAccessibilityBridge::ClearFlutterSemanticsCaches() g_flutterSemanticsTree.clear(); g_parentChildIdVec.clear(); g_screenRectMap.clear(); + Flutter_SendAccessibilityAsyncEvent( + accessibilityFocusedNode.id, + ArkUI_AccessibilityEventType::ARKUI_ACCESSIBILITY_NATIVE_EVENT_TYPE_ACCESSIBILITY_FOCUS_CLEARED); + accessibilityFocusedNode = {}; } /** diff --git a/shell/platform/ohos/accessibility/ohos_accessibility_bridge.h b/shell/platform/ohos/accessibility/ohos_accessibility_bridge.h index d202f7e6ec..673f762d37 100644 --- a/shell/platform/ohos/accessibility/ohos_accessibility_bridge.h +++ b/shell/platform/ohos/accessibility/ohos_accessibility_bridge.h @@ -30,6 +30,9 @@ #include "ohos_accessibility_ddl.h" #include "flutter/shell/platform/ohos/utils/ohos_utils.h" #include "flutter/shell/platform/ohos/utils/arkui_accessibility_constant.h" +#include "third_party/skia/include/core/SkMatrix.h" +#include "third_party/skia/include/core/SkScalar.h" +#include "third_party/skia/include/core/SkPoint.h" namespace flutter { typedef flutter::SemanticsFlags FLAGS_; @@ -43,9 +46,15 @@ struct AbsoluteRect { static constexpr AbsoluteRect MakeEmpty() { return AbsoluteRect{0.0, 0.0, 0.0, 0.0}; } + static constexpr AbsoluteRect MakeRect( + float left, float top, float right, float bottom) { + return AbsoluteRect{left, top, right, bottom}; + } + }; struct SemanticsNodeExtent : flutter::SemanticsNode { bool isNull = false; + SkM44 globalTransform = SkM44{}; AbsoluteRect absoluteRect = AbsoluteRect::MakeEmpty(); int32_t parentId = -1; bool hadPreviousConfig = false; @@ -81,12 +90,9 @@ public: std::string xcomponentId_; bool isFlutterNavigated_; int64_t native_shell_holder_id_; - ArkUI_AccessibilityProvider* provider_; void OnOhosAccessibilityStateChange(bool ohosAccessibilityEnabled, int64_t shellholderId); - void SetNativeShellHolderId(int64_t id); - void UpdateSemantics(flutter::SemanticsNodeUpdates update, flutter::CustomAccessibilityActionUpdates actions); @@ -139,7 +145,7 @@ public: std::unique_ptr& message, ArkUI_AccessibilityEventType eventType); - void FlutterRelativeRectToScreenRect(SemanticsNodeExtent node); + void RelativeRectToScreenRect(SemanticsNodeExtent& node); AbsoluteRect GetAbsoluteScreenRect(SemanticsNodeExtent& flutterNode); void SetAbsoluteScreenRect(SemanticsNodeExtent& flutterNode, float left, @@ -169,6 +175,7 @@ private: std::unordered_map g_flutterSemanticsTree; std::vector> g_parentChildIdVec; std::unordered_map g_screenRectMap; + std::unordered_map g_globalTransformMap; SemanticsNodeExtent inputFocusedNode; SemanticsNodeExtent lastInputFocusedNode; @@ -255,6 +262,11 @@ private: flutter::SemanticsAction ArkuiActionsToFlutterActions( ArkUI_Accessibility_ActionType arkui_action); + void BuildParentChildNodeIdRelation(const SemanticsNodeExtent& node); + void ComputeGlobalTransform(); + void ConvertRectToGlobal(SemanticsNodeExtent& node); + SkPoint ApplyTransform(SkPoint& point, const SkM44& transform); + bool HasScrolled(const SemanticsNodeExtent& flutterNode); bool HasChangedLabel(const SemanticsNodeExtent& flutterNode); diff --git a/shell/platform/ohos/ohos_xcomponent_adapter.cpp b/shell/platform/ohos/ohos_xcomponent_adapter.cpp index c130995549..68d1a5debf 100644 --- a/shell/platform/ohos/ohos_xcomponent_adapter.cpp +++ b/shell/platform/ohos/ohos_xcomponent_adapter.cpp @@ -99,10 +99,6 @@ void XComponentAdapter::AttachFlutterEngine(std::string& id, auto findIter = xcomponetMap_.find(id); if (findIter != xcomponetMap_.end()) { findIter->second->AttachFlutterEngine(shellholderId); - // register the OH_ArkUI accessibility callbacks - if (OH_GetSdkApiVersion() >= 13) { - findIter->second->RegisterArkUIAccessibilityService(findIter->second->nativeXComponent_, shellholderId); - } } } @@ -199,6 +195,7 @@ void OnSurfaceChangedCB(OH_NativeXComponent* component, void* window) { } void OnSurfaceDestroyedCB(OH_NativeXComponent* component, void* window) { + std::lock_guard lock(XComponentAdapter::GetInstance()->mutex_); for(auto it = XComponentAdapter::GetInstance()->xcomponetMap_.begin(); it != XComponentAdapter::GetInstance()->xcomponetMap_.end();) { @@ -370,8 +367,17 @@ void XComponentBase::DetachFlutterEngine() { isEngineAttached_ = false; } -void XComponentBase::RegisterArkUIAccessibilityService( - OH_NativeXComponent* nativeXComponent, const std::string& shellholderId) +ArkUI_AccessibilityProvider* XComponentAdapter::GetAccessibilityProvider(const std::string& xcompId) +{ + auto it = xcomponetMap_.find(xcompId); + if (it != xcomponetMap_.end()) { + return it->second->accessibilityProvider_; + } else { + return nullptr; + } +} + +void XComponentBase::RegisterArkUIAccessibilityService(OH_NativeXComponent* nativeXComponent) { BindAccessibilityProviderCallback(); @@ -392,8 +398,8 @@ void XComponentBase::RegisterArkUIAccessibilityService( OH_ArkUI_AccessibilityProviderRegisterCallback(a11yProvider, &accessibilityProviderCallback_) ); + std::lock_guard lock(XComponentAdapter::GetInstance()->mutex_); auto* base = XComponentAdapter::GetInstance()->xcomponetMap_[id_]; - base->shellholderId_ = shellholderId; base->accessibilityProvider_ = a11yProvider; base->nativeXComponent_ = nativeXComponent; @@ -406,6 +412,9 @@ void XComponentBase::SetNativeXComponent(OH_NativeXComponent* nativeXComponent){ BindXComponentCallback(); OH_NativeXComponent_RegisterCallback(nativeXComponent_, &callback_); OH_NativeXComponent_RegisterMouseEventCallback(nativeXComponent_, &mouseCallback_); + // register the OH_ArkUI accessibility callbacks + if (OH_GetSdkApiVersion() < 13) { return; } + RegisterArkUIAccessibilityService(nativeXComponent_); } } @@ -462,8 +471,6 @@ void XComponentBase::OnSurfaceChanged(OH_NativeXComponent* component, void* wind void XComponentBase::OnSurfaceDestroyed(OH_NativeXComponent* component, void* window) { window_ = nullptr; - // XComponentAdapter::GetInstance()->accessibilityProvider_ = nullptr; - LOGD("XComponentManger::OnSurfaceDestroyed"); if (isEngineAttached_) { PlatformViewOHOSNapi::SurfaceDestroyed(std::stoll(shellholderId_)); diff --git a/shell/platform/ohos/ohos_xcomponent_adapter.h b/shell/platform/ohos/ohos_xcomponent_adapter.h index 3500fabdb4..632e9d216d 100644 --- a/shell/platform/ohos/ohos_xcomponent_adapter.h +++ b/shell/platform/ohos/ohos_xcomponent_adapter.h @@ -20,7 +20,7 @@ #include #include #include - +#include #include "flutter/shell/platform/ohos/ohos_touch_processor.h" #include "flutter/shell/platform/ohos/napi/platform_view_ohos_napi.h" #include "napi/native_api.h" @@ -52,7 +52,7 @@ public: void OnDispatchMouseWheelEvent(mouseWheelEvent event); void RegisterArkUIAccessibilityService( - OH_NativeXComponent* nativeXComponent, const std::string& shellholderId); + OH_NativeXComponent* nativeXComponent); OH_NativeXComponent_TouchEvent touchEvent_; OH_NativeXComponent_Callback callback_; @@ -83,9 +83,12 @@ class XComponentAdapter { void DetachFlutterEngine(std::string& id); void OnMouseWheel(std::string& id, mouseWheelEvent event); + ArkUI_AccessibilityProvider* GetAccessibilityProvider(const std::string& xcompId); + public: std::map xcomponetMap_; std::string currentXComponentId_; + std::mutex mutex_; private: static XComponentAdapter mXComponentAdapter; -- Gitee