Add perform action for Eche accessibility.
- Adds conversion from ui:AXActionData to proto data. Skips mojom types.
- Todos will be resolved when this is integrated with the accessibility
system in chrome.
See Design at:
go/exo-accessibility-support
Bug: b/267373987
Change-Id: Id7ce464d0134773e22596ac1f9a755c2709b096c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4518144
Reviewed-by: Matthew Denton <mpdenton@chromium.org>
Reviewed-by: Abbas Nayebi <nayebi@google.com>
Commit-Queue: Jacob Francis <francisjp@google.com>
Cr-Commit-Position: refs/heads/main@{#1145673}
diff --git a/ash/webui/eche_app_ui/accessibility_provider.cc b/ash/webui/eche_app_ui/accessibility_provider.cc
index ee3b37c..b95251e2 100644
--- a/ash/webui/eche_app_ui/accessibility_provider.cc
+++ b/ash/webui/eche_app_ui/accessibility_provider.cc
@@ -4,8 +4,8 @@
// found in the LICENSE file.
#include "ash/webui/eche_app_ui/accessibility_provider.h"
+
#include <cstdint>
-#include "ash/webui/eche_app_ui/accessibility_tree_converter.h"
#include "base/notreached.h"
namespace ash::eche_app {
@@ -21,6 +21,25 @@
NOTIMPLEMENTED();
}
+void AccessibilityProvider::SetAccessibilityObserver(
+ mojo::PendingRemote<mojom::AccessibilityObserver> observer) {
+ observer_remote_.reset();
+ observer_remote_.Bind(std::move(observer));
+}
+
+void AccessibilityProvider::PerformAction(const ui::AXActionData& action) {
+ AccessibilityTreeConverter converter;
+ auto proto_action = converter.ConvertActionDataToProto(action);
+ if (proto_action.has_value()) {
+ size_t nbytes = proto_action->ByteSizeLong();
+ std::vector<uint8_t> serialized_proto(nbytes);
+ proto_action->SerializeToArray(serialized_proto.data(), nbytes);
+ observer_remote_->PerformAction(serialized_proto);
+ } else {
+ LOG(ERROR) << "Failed to serialize AXActionData to protobuf.";
+ }
+}
+
void AccessibilityProvider::Bind(
mojo::PendingReceiver<mojom::AccessibilityProvider> receiver) {
receiver_.reset();
diff --git a/ash/webui/eche_app_ui/accessibility_provider.h b/ash/webui/eche_app_ui/accessibility_provider.h
index 73dfe815..486bd08 100644
--- a/ash/webui/eche_app_ui/accessibility_provider.h
+++ b/ash/webui/eche_app_ui/accessibility_provider.h
@@ -5,11 +5,13 @@
#ifndef ASH_WEBUI_ECHE_APP_UI_ACCESSIBILITY_PROVIDER_H_
#define ASH_WEBUI_ECHE_APP_UI_ACCESSIBILITY_PROVIDER_H_
+#include "ash/webui/eche_app_ui/accessibility_tree_converter.h"
#include "ash/webui/eche_app_ui/mojom/eche_app.mojom.h"
#include "mojo/public/cpp/bindings/pending_remote.h"
#include "mojo/public/cpp/bindings/receiver.h"
#include "mojo/public/cpp/bindings/remote.h"
+#include "ui/accessibility/ax_action_data.h"
namespace ash::eche_app {
class AccessibilityProvider : public mojom::AccessibilityProvider {
@@ -19,10 +21,15 @@
// Proto from ash/webui/eche_app_ui/proto/accessibility_mojom.proto.
void HandleAccessibilityEventReceived(
const std::vector<uint8_t>& serialized_proto) override;
+ void SetAccessibilityObserver(
+ ::mojo::PendingRemote<mojom::AccessibilityObserver> observer) override;
void Bind(mojo::PendingReceiver<mojom::AccessibilityProvider> receiver);
+ void PerformAction(const ui::AXActionData& action);
+
private:
mojo::Receiver<mojom::AccessibilityProvider> receiver_{this};
+ mojo::Remote<mojom::AccessibilityObserver> observer_remote_;
};
} // namespace ash::eche_app
#endif // ASH_WEBUI_ECHE_APP_UI_ACCESSIBILITY_PROVIDER_H_
diff --git a/ash/webui/eche_app_ui/accessibility_tree_converter.cc b/ash/webui/eche_app_ui/accessibility_tree_converter.cc
index 2a99fc34..b7032f7 100644
--- a/ash/webui/eche_app_ui/accessibility_tree_converter.cc
+++ b/ash/webui/eche_app_ui/accessibility_tree_converter.cc
@@ -18,6 +18,7 @@
#include "base/functional/callback_helpers.h"
#include "base/notreached.h"
#include "mojo/public/cpp/bindings/struct_ptr.h"
+#include "ui/accessibility/ax_enums.mojom-shared.h"
namespace ash::eche_app {
AccessibilityTreeConverter::AccessibilityTreeConverter() = default;
@@ -164,6 +165,138 @@
return out_data;
}
+absl::optional<proto::AccessibilityActionType> ConvertType(
+ ax::mojom::Action action_type) {
+ switch (action_type) {
+ case ax::mojom::Action::kDoDefault:
+ return proto::AccessibilityActionType::ACTION_CLICK;
+ case ax::mojom::Action::kFocus:
+ return proto::AccessibilityActionType::ACTION_ACCESSIBILITY_ACTION_FOCUS;
+ case ax::mojom::Action::kSetSequentialFocusNavigationStartingPoint:
+ return proto::AccessibilityActionType::ACTION_ACCESSIBILITY_FOCUS;
+ case ax::mojom::Action::kScrollToMakeVisible:
+ return proto::AccessibilityActionType::ACTION_SHOW_ON_SCREEN;
+ case ax::mojom::Action::kScrollBackward:
+ return proto::AccessibilityActionType::ACTION_SCROLL_BACKWARD;
+ case ax::mojom::Action::kScrollForward:
+ return proto::AccessibilityActionType::ACTION_SCROLL_FORWARD;
+ case ax::mojom::Action::kScrollUp:
+ return proto::AccessibilityActionType::ACTION_SCROLL_UP;
+ case ax::mojom::Action::kScrollDown:
+ return proto::AccessibilityActionType::ACTION_SCROLL_DOWN;
+ case ax::mojom::Action::kScrollLeft:
+ return proto::AccessibilityActionType::ACTION_SCROLL_LEFT;
+ case ax::mojom::Action::kScrollRight:
+ return proto::AccessibilityActionType::ACTION_SCROLL_RIGHT;
+ case ax::mojom::Action::kScrollToPositionAtRowColumn:
+ return proto::AccessibilityActionType::ACTION_SCROLL_TO_POSITION;
+ case ax::mojom::Action::kCustomAction:
+ return proto::AccessibilityActionType::ACTION_CUSTOM_ACTION;
+ case ax::mojom::Action::kSetAccessibilityFocus:
+ return proto::AccessibilityActionType::ACTION_ACCESSIBILITY_FOCUS;
+ case ax::mojom::Action::kClearAccessibilityFocus:
+ return proto::AccessibilityActionType::ACTION_CLEAR_ACCESSIBILITY_FOCUS;
+ case ax::mojom::Action::kGetTextLocation:
+ return proto::AccessibilityActionType::ACTION_GET_TEXT_LOCATION;
+ case ax::mojom::Action::kShowTooltip:
+ return proto::AccessibilityActionType::ACTION_SHOW_TOOLTIP;
+ case ax::mojom::Action::kHideTooltip:
+ return proto::AccessibilityActionType::ACTION_HIDE_TOOLTIP;
+ case ax::mojom::Action::kCollapse:
+ return proto::AccessibilityActionType::ACTION_COLLAPSE;
+ case ax::mojom::Action::kExpand:
+ return proto::AccessibilityActionType::ACTION_EXPAND;
+ case ax::mojom::Action::kLongClick:
+ return proto::AccessibilityActionType::ACTION_LONG_CLICK;
+ default:
+ return absl::nullopt;
+ }
+}
+
+void PopulateActionParameters(const ui::AXActionData& chrome_data,
+ proto::AccessibilityActionData& action_data) {
+ switch (action_data.action_type()) {
+ case proto::AccessibilityActionType::ACTION_SCROLL_TO_POSITION: {
+ const auto [row, column] = chrome_data.row_column;
+ auto* row_kvp = action_data.add_int_parameters();
+ row_kvp->set_key(proto::ActionIntArgumentType::TYPE_ROW_INT);
+ row_kvp->set_value(row);
+ auto* col_kvp = action_data.add_int_parameters();
+ col_kvp->set_key(proto::ActionIntArgumentType::TYPE_COLUMN_INT);
+ col_kvp->set_value(column);
+ break;
+ }
+ case proto::AccessibilityActionType::ACTION_CUSTOM_ACTION:
+ action_data.set_custom_action_id(chrome_data.custom_action_id);
+ break;
+ case proto::AccessibilityActionType::ACTION_NEXT_HTML_ELEMENT:
+ case proto::AccessibilityActionType::ACTION_PREVIOUS_HTML_ELEMENT:
+ case proto::AccessibilityActionType::ACTION_ACCESSIBILITY_ACTION_FOCUS:
+ case proto::AccessibilityActionType::ACTION_CLEAR_FOCUS:
+ case proto::AccessibilityActionType::ACTION_SELECT:
+ case proto::AccessibilityActionType::ACTION_CLEAR_SELECTION:
+ case proto::AccessibilityActionType::ACTION_CLICK:
+ case proto::AccessibilityActionType::ACTION_LONG_CLICK:
+ case proto::AccessibilityActionType::ACTION_ACCESSIBILITY_FOCUS:
+ case proto::AccessibilityActionType::ACTION_CLEAR_ACCESSIBILITY_FOCUS:
+ case proto::AccessibilityActionType::ACTION_NEXT_AT_MOVEMENT_GRANULARITY:
+ case proto::AccessibilityActionType::
+ ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY:
+ case proto::AccessibilityActionType::ACTION_SCROLL_FORWARD:
+ case proto::AccessibilityActionType::ACTION_SCROLL_BACKWARD:
+ case proto::AccessibilityActionType::ACTION_COPY:
+ case proto::AccessibilityActionType::ACTION_PASTE:
+ case proto::AccessibilityActionType::ACTION_CUT:
+ case proto::AccessibilityActionType::ACTION_SET_SELECTION:
+ case proto::AccessibilityActionType::ACTION_EXPAND:
+ case proto::AccessibilityActionType::ACTION_COLLAPSE:
+ case proto::AccessibilityActionType::ACTION_DISMISS:
+ case proto::AccessibilityActionType::ACTION_SET_TEXT:
+ case proto::AccessibilityActionType::ACTION_CONTEXT_CLICK:
+ case proto::AccessibilityActionType::ACTION_SCROLL_DOWN:
+ case proto::AccessibilityActionType::ACTION_SCROLL_LEFT:
+ case proto::AccessibilityActionType::ACTION_SCROLL_RIGHT:
+ case proto::AccessibilityActionType::ACTION_SCROLL_UP:
+ case proto::AccessibilityActionType::ACTION_SET_PROGRESS:
+ case proto::AccessibilityActionType::ACTION_SHOW_ON_SCREEN:
+ case proto::AccessibilityActionType::ACTION_GET_TEXT_LOCATION:
+ case proto::AccessibilityActionType::ACTION_SHOW_TOOLTIP:
+ case proto::AccessibilityActionType::ACTION_HIDE_TOOLTIP:
+ case proto::AccessibilityActionType::
+ AccessibilityActionType_INT_MIN_SENTINEL_DO_NOT_USE_:
+ case proto::AccessibilityActionType::
+ AccessibilityActionType_INT_MAX_SENTINEL_DO_NOT_USE_:
+ break;
+ }
+}
+
+absl::optional<proto::AccessibilityActionData>
+AccessibilityTreeConverter::ConvertActionDataToProto(
+ const ui::AXActionData& data) {
+ proto::AccessibilityActionData action_data;
+ // TODO(francisjp/282044350) window_id - We may not actually end up having a
+ // window id since there is only one window at a time.
+ auto action_type = ConvertType(data.action);
+ if (!action_type.has_value()) {
+ return absl::nullopt;
+ }
+ action_data.set_action_type(action_type.value());
+ action_data.set_node_id(data.target_node_id);
+ PopulateActionParameters(data, action_data);
+
+ if (action_type == proto::AccessibilityActionType::ACTION_GET_TEXT_LOCATION) {
+ action_data.set_start_index(data.start_index);
+ action_data.set_end_index(data.end_index);
+ // TODO(francisjp/282044350) Refresh with Extra Data here
+ bool refresh_success = true;
+ if (!refresh_success) {
+ return absl::nullopt;
+ }
+ }
+
+ return action_data;
+}
+
// Object converters
mojo::StructPtr<AXWindowData> AccessibilityTreeConverter::ToMojomWindowData(
const proto::AccessibilityWindowInfoData& proto_in) {
diff --git a/ash/webui/eche_app_ui/accessibility_tree_converter.h b/ash/webui/eche_app_ui/accessibility_tree_converter.h
index 4c402bc..1cfd493 100644
--- a/ash/webui/eche_app_ui/accessibility_tree_converter.h
+++ b/ash/webui/eche_app_ui/accessibility_tree_converter.h
@@ -9,6 +9,7 @@
#include "ash/components/arc/mojom/accessibility_helper.mojom.h"
#include "ash/webui/eche_app_ui/proto/accessibility_mojom.pb.h"
+#include "ui/accessibility/ax_action_data.h"
namespace {
@@ -54,6 +55,9 @@
mojo::StructPtr<AXEventData> ConvertEventDataProtoToMojom(
const std::vector<uint8_t>& serialized_proto);
+ absl::optional<proto::AccessibilityActionData> ConvertActionDataToProto(
+ const ui::AXActionData& data);
+
private:
// Utility Functions
template <class ProtoType, class MojomType>
diff --git a/ash/webui/eche_app_ui/mojom/eche_app.mojom b/ash/webui/eche_app_ui/mojom/eche_app.mojom
index 9038fa9..db58b1fb 100644
--- a/ash/webui/eche_app_ui/mojom/eche_app.mojom
+++ b/ash/webui/eche_app_ui/mojom/eche_app.mojom
@@ -74,6 +74,18 @@
// the browser.
// Proto from: ash/webui/eche_app_ui/proto/accessibility_mojom.proto
HandleAccessibilityEventReceived(array<uint8> serialized_proto);
+
+ // Sets the observer for when accessibility actions need to be send to the
+ // SWA from the browser.
+ SetAccessibilityObserver(pending_remote<AccessibilityObserver> observer);
+};
+
+// Observer for accessibility provider. Should be created using
+// SetAccessibilityObserver in AccessibilityProvider.
+interface AccessibilityObserver {
+ // Sends the accessibility action to the SWA for processing.
+ // Proto from: ash/webui/eche_app_ui/proto/accessibility_mojom.proto
+ PerformAction(array<uint8> serialized_proto);
};
// Interface for generating uid. The uid is unique and persistent.
diff --git a/ash/webui/eche_app_ui/resources/browser_proxy.js b/ash/webui/eche_app_ui/resources/browser_proxy.js
index 9674721..103fbe3 100644
--- a/ash/webui/eche_app_ui/resources/browser_proxy.js
+++ b/ash/webui/eche_app_ui/resources/browser_proxy.js
@@ -43,6 +43,13 @@
systemInfoObserverRouter.$.bindNewPipeAndPassRemote());
// Returns the remote for AccessibilityProvider.
const accessibility = ash.echeApp.mojom.AccessibilityProvider.getRemote();
+// An object that receives requests for the AccessibilityObserver mojom
+// interface and dispatches them as callbacks. Setup the message
+const accessibilityObserverRouter =
+ new ash.echeApp.mojom.AccessibilityObserverCallbackRouter();
+// Set up a message pipe to the browser process to accessibility actions.
+accessibility.setAccessibilityObserver(
+ accessibilityObserverRouter.$.bindNewPipeAndPassRemote());
const notificationGenerator =
ash.echeApp.mojom.NotificationGenerator.getRemote();
@@ -148,6 +155,10 @@
});
});
+accessibilityObserverRouter.performAction.addListener(action => {
+ guestMessagePipe.sendMessage(Message.ACCESSIBILITY_PERFORM_ACTION, action);
+});
+
// Add stream action listener and send result via pipes.
streamActionObserverRouter.onStreamAction.addListener((action) => {
console.log(`echeapi browser_proxy.js OnStreamAction ${action}`);
diff --git a/ash/webui/eche_app_ui/resources/message_types.js b/ash/webui/eche_app_ui/resources/message_types.js
index dcc3bf5..8251827c 100644
--- a/ash/webui/eche_app_ui/resources/message_types.js
+++ b/ash/webui/eche_app_ui/resources/message_types.js
@@ -116,4 +116,6 @@
CONNECTION_STATUS_CHANGED: 'connection_status_changed',
// Message for sending accessibility event data.
ACCESSIBILITY_EVENT_DATA: 'accessibility_event_data',
+ // Message for sending actions and their parameters.
+ ACCESSIBILITY_PERFORM_ACTION: 'accessibility_perform_action',
};
diff --git a/ash/webui/eche_app_ui/resources/receiver.js b/ash/webui/eche_app_ui/resources/receiver.js
index 92cfbf7..374665d 100644
--- a/ash/webui/eche_app_ui/resources/receiver.js
+++ b/ash/webui/eche_app_ui/resources/receiver.js
@@ -69,6 +69,12 @@
/** @type {boolean} */ (message.androidDeviceOnCellular));
});
+// Handle accessibility perform action.
+parentMessagePipe.registerHandler(
+ Message.ACCESSIBILITY_PERFORM_ACTION, async (action) => {
+ console.log('Performed accessibility action: ' + action);
+ });
+
// The implementation of echeapi.d.ts
const EcheApiBindingImpl = new (class {
closeWindow() {
