Remove deprecated UIComponent from chrome_screen_ai.proto
`UIComponent` was used to send layout extraction results from
ChromeScreenAI library and is now deprecated. The field is removed in
Google3 code in cl/646470338.
AX-Relnotes: n/a.
Bug: 313384385
Change-Id: I2ee308fac6f9375be5fe90510a4e1059e57fdc97
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5654251
Commit-Queue: Ramin Halavati <rhalavati@chromium.org>
Reviewed-by: Kyungjun Lee <kyungjunlee@google.com>
Cr-Commit-Position: refs/heads/main@{#1319571}
diff --git a/services/screen_ai/proto/chrome_screen_ai.proto b/services/screen_ai/proto/chrome_screen_ai.proto
index 654ad51..deb0acb 100644
--- a/services/screen_ai/proto/chrome_screen_ai.proto
+++ b/services/screen_ai/proto/chrome_screen_ai.proto
@@ -10,240 +10,6 @@
package chrome_screen_ai;
-// A component of a given UI. This proto is filled based on
-// /research/socrates/proto/visual.proto. Refer to that for more context.
-//
-// Next available field is 3.
-message UIComponent {
- // This enum should be in sync with ax::mojom::Role in
- // https://source.chromium.org/chromium/chromium/src/+/main:ui/accessibility/ax_enums.mojom.
- enum Type {
- NONE = 0;
- ABBR = 1;
- ALERT = 2;
- ALERT_DIALOG = 3;
- APPLICATION = 4;
- ARTICLE = 5;
- AUDIO = 6;
- BANNER = 7;
- BLOCK_QUOTE = 8;
- BUTTON = 9;
- CANVAS = 10;
- CAPTION = 11;
- CARET = 12;
- CELL = 13;
- CHECKBOX = 14;
- CLIENT = 15;
- CODE = 16;
- COLOR_WELL = 17;
- COLUMN = 18;
- COLUMN_HEADER = 19;
- COMBOBOX_GROUPING = 20;
- COMBOBOX_MENU_BUTTON = 21;
- COMPLEMENTARY = 22;
- COMMENT = 23;
- CONTENT_DELETION = 24;
- CONTENT_INSERTION = 25;
- CONTENT_INFO = 26;
- DATE = 27;
- DATE_TIME = 28;
- DEFINITION = 29;
- DESCRIPTION_LIST = 30;
- DESCRIPTION_LIST_DETAIL = 31;
- DESCRIPTION_LIST_TERM = 32;
- DESKTOP = 33;
- DETAILS = 34;
- DIALOG = 35;
- DIRECTORY = 36;
- DISCLOSURE_TRIANGLE = 37;
- DOC_ABSTRACT = 38;
- DOC_ACKNOWLEDGMENTS = 39;
- DOC_AFTERWORD = 40;
- DOC_APPENDIX = 41;
- DOC_BACKLINK = 42;
- DOC_BIBLIOENTRY = 43;
- DOC_BIBLIOGRAPHY = 44;
- DOC_BIBLIOREF = 45;
- DOC_CHAPTER = 46;
- DOC_COLOPHON = 47;
- DOC_CONCLUSION = 48;
- DOC_COVER = 49;
- DOC_CREDIT = 50;
- DOC_CREDITS = 51;
- DOC_DEDICATION = 52;
- DOC_END_NOTE = 53;
- DOC_END_NOTES = 54;
- DOC_EPIGRAPH = 55;
- DOC_EPILOGUE = 56;
- DOC_ERRATA = 57;
- DOC_EXAMPLE = 58;
- DOC_FOOTNOTE = 59;
- DOC_FOREWORD = 60;
- DOC_GLOSSARY = 61;
- DOC_GLOSS_REF = 62;
- DOC_INDEX = 63;
- DOC_INTRODUCTION = 64;
- DOC_NOTE_REF = 65;
- DOC_NOTICE = 66;
- DOC_PAGE_BREAK = 67;
- DOC_PAGE_FOOTER = 68;
- DOC_PAGE_HEADER = 69;
- DOC_PAGELIST = 70;
- DOC_PART = 71;
- DOC_PREFACE = 72;
- DOC_PROLOGUE = 73;
- DOC_PULL_QUOTE = 74;
- DOC_QNA = 75;
- DOC_SUBTITLE = 76;
- DOC_TIP = 77;
- DOC_TOC = 78;
- DOCUMENT = 79;
- EMBEDDEDOBJECT = 80;
- EMPHASIS = 81;
- FEED = 82;
- FIG_CAPTION = 83;
- FIGURE = 84;
- FOOTER = 85;
- FOOTER_AS_NON_LANDMARK = 86;
- FORM = 87;
- GENERIC_CONTAINER = 88;
- GRAPHICS_DOCUMENT = 89;
- GRAPHICS_OBJECT = 90;
- GRAPHICS_SYMBOL = 91;
- GRID = 92;
- GROUP = 93;
- HEADER = 94;
- HEADERAS_NON_LANDMARK = 95;
- HEADING = 96;
- IFRAME = 97;
- IFRAME_PRESENTATIONAL = 98;
- IMAGE = 99;
- IME_CANDIDATE = 100;
- INLINE_TEXTBOX = 101;
- INPUT_TIME = 102;
- KEYBOARD = 103;
- LABELTEXT = 104;
- LAYOUT_TABLE = 105;
- LAYOUT_TABLE_CELL = 106;
- LAYOUT_TABLE_ROW = 107;
- LEGEND = 108;
- LINEBREAK = 109;
- LINK = 110;
- LIST = 111;
- LISTBOX = 112;
- LISTBOX_OPTION = 113;
- LIST_GRID = 114;
- LIST_ITEM = 115;
- LIST_MARKER = 116;
- LOG = 117;
- MAIN = 118;
- MARK = 119;
- MARQUEE = 120;
- MATH = 121;
- MATHML_FRACTION = 188;
- MATHML_IDENTIFIER = 189;
- MATHML_MATH = 187;
- MATHML_MULTISCRIPTS = 190;
- MATHML_NONESCRIPT = 191;
- MATHML_NUMBER = 192;
- MATHML_OPERATOR = 193;
- MATHML_OVER = 194;
- MATHML_PRESCRIPTDELIMITER = 195;
- MATHML_ROOT = 196;
- MATHML_ROW = 197;
- MATHML_SQUAREROOT = 198;
- MATHML_STRINGLITERAL = 199;
- MATHML_SUB = 200;
- MATHML_SUBSUP = 201;
- MATHML_SUP = 202;
- MATHML_TABLE = 203;
- MATHML_TABLECELL = 204;
- MATHML_TABLEROW = 205;
- MATHML_TEXT = 206;
- MATHML_UNDER = 207;
- MATHML_UNDEROVER = 208;
- MENU = 122;
- MENU_BAR = 123;
- MENU_ITEM = 124;
- MENU_ITEM_CHECKBOX = 125;
- MENU_ITEM_RADIO = 126;
- MENU_LIST_OPTION = 127;
- MENU_LIST_POPUP = 128;
- METER = 129;
- NAVIGATION = 130;
- NOTE = 131;
- PANE = 132;
- PARAGRAPH = 133;
- PDF_ACTIONABLE_HIGHLIGHT = 134;
- PDF_ROOT = 135;
- PLUGIN_OBJECT = 136;
- POPUP_BUTTON = 137;
- PORTAL = 138;
- PRE = 139;
- PROGRESS_INDICATOR = 140;
- RADIO_BUTTON = 141;
- RADIO_GROUP = 142;
- REGION = 143;
- ROOT_WEB_AREA = 144;
- ROW = 145;
- ROW_GROUP = 146;
- ROW_HEADER = 147;
- RUBY = 148;
- RUBY_ANNOTATION = 149;
- SCROLL_BAR = 150;
- SCROLL_VIEW = 151;
- SEARCH = 152;
- SEARCHBOX = 153;
- SECTION = 154;
- SLIDER = 155;
- SPIN_BUTTON = 156;
- SPLITTER = 157;
- STATIC_TEXT = 158;
- STATUS = 159;
- STRONG = 160;
- SUBSCRIPT = 185;
- SUGGESTION = 161;
- SUPERSCRIPT = 186;
- SVGROOT = 162;
- SWITCH = 163;
- TAB = 164;
- TABLIST = 165;
- TABPANEL = 166;
- TABLE = 167;
- TABLE_HEADER_CONTAINER = 168;
- TERM = 169;
- TEXT_FIELD = 170;
- TEXT_FIELD_WITH_COMBOBOX = 171;
- TIME = 172;
- TIMER = 173;
- TITLEBAR = 174;
- TOGGLE_BUTTON = 175;
- TOOLBAR = 176;
- TOOLTIP = 177;
- TREE = 178;
- TREE_GRID = 179;
- TREE_ITEM = 180;
- UNKNOWN = 181;
- VIDEO = 182;
- WEBVIEW = 183;
- WINDOW = 184;
- }
-
- // The predicted type of this component.
- message PredictedType {
- oneof type_of {
- Type enum_type = 1;
- string string_type = 2;
- }
- }
-
- PredictedType predicted_type = 1;
-
- // The on-screen bounding box for this component. Origin (0,0) is the top-left
- // corner.
- Rect bounding_box = 2;
-}
-
// Defines a rectangle.
message Rect {
int32 x = 1;
@@ -385,6 +151,7 @@
}
message VisualAnnotation {
- repeated UIComponent ui_component = 1;
+ // `ui_component` deprecated in Chrome M128.
+ reserved 1;
repeated LineBox lines = 2;
}
diff --git a/services/screen_ai/proto/visual_annotator_proto_convertor.cc b/services/screen_ai/proto/visual_annotator_proto_convertor.cc
index a1338e3..2ce5ef1 100644
--- a/services/screen_ai/proto/visual_annotator_proto_convertor.cc
+++ b/services/screen_ai/proto/visual_annotator_proto_convertor.cc
@@ -85,32 +85,6 @@
return true;
}
-// Returns whether the provided `predicted_type` is:
-// A) set, and
-// B) has a confidence that is above our acceptance threshold.
-bool SerializePredictedType(
- const chrome_screen_ai::UIComponent::PredictedType& predicted_type,
- ui::AXNodeData& out_data) {
- DCHECK_EQ(out_data.role, ax::mojom::Role::kUnknown);
- switch (predicted_type.type_of_case()) {
- case chrome_screen_ai::UIComponent::PredictedType::kEnumType:
- out_data.role = static_cast<ax::mojom::Role>(predicted_type.enum_type());
- break;
- case chrome_screen_ai::UIComponent::PredictedType::kStringType:
- out_data.role = ax::mojom::Role::kGenericContainer;
- out_data.AddStringAttribute(ax::mojom::StringAttribute::kRoleDescription,
- predicted_type.string_type());
- break;
- case chrome_screen_ai::UIComponent::PredictedType::TYPE_OF_NOT_SET:
- NOTREACHED_IN_MIGRATION()
- << "Malformed proto message: Required field "
- "`chrome_screen_ai::UIComponent::PredictedType` not set.";
- return false;
- }
-
- return true;
-}
-
void SerializeBoundingBox(const chrome_screen_ai::Rect& bounding_box,
const ui::AXNodeID& container_id,
ui::AXNodeData& out_data) {
@@ -380,21 +354,6 @@
return 1u;
}
-void SerializeUIComponent(const chrome_screen_ai::UIComponent& ui_component,
- const size_t index,
- ui::AXNodeData& parent_node,
- std::vector<ui::AXNodeData>& node_data) {
- DCHECK_LT(index, node_data.size());
- DCHECK_NE(parent_node.id, ui::kInvalidAXNodeID);
- ui::AXNodeData& current_node = node_data[index];
- if (!SerializePredictedType(ui_component.predicted_type(), current_node))
- return;
- current_node.id = GetNextNegativeNodeID();
- SerializeBoundingBox(ui_component.bounding_box(), parent_node.id,
- current_node);
- parent_node.child_ids.push_back(current_node.id);
-}
-
// Returns the number of accessibility nodes that have been initialized in
// `node_data`. A single `line_box` may turn into a number of inline text boxes
// depending on how many formatting contexts it contains. If `line_box` is of a
@@ -473,8 +432,9 @@
const gfx::Rect& image_rect) {
ui::AXTreeUpdate update;
- DCHECK(visual_annotation.lines_size() == 0 ||
- visual_annotation.ui_component_size() == 0);
+ if (visual_annotation.lines().empty()) {
+ return update;
+ }
// TODO(https://crbug.com/327298772): Create an AXTreeSource and create the
// update using AXTreeSerializer.
@@ -517,110 +477,90 @@
std::make_pair(line.order_within_block(), i));
}
- size_t rootnodes_count = 0u;
- if (!visual_annotation.ui_component().empty())
- ++rootnodes_count;
- if (!visual_annotation.lines().empty()) {
- ++rootnodes_count;
- // Need four more nodes that convey the disclaimer messages. There are two
- // messages, one before the content and one after. Each message is wrapped
- // in an ARIA landmark so that it can easily be navigated to by a screen
- // reader user and thus not missed.
- formatting_context_count += 4;
- }
+ // Need four more nodes that convey the disclaimer messages. There are two
+ // messages, one before the content and one after. Each message is wrapped
+ // in an ARIA landmark so that it can easily be navigated to by a screen
+ // reader user and thus not missed.
+ formatting_context_count += 4;
// There are the same number of paragraphs as blocks.
size_t paragraph_count = blocks_to_lines_map.size();
- std::vector<ui::AXNodeData> nodes(rootnodes_count +
- visual_annotation.ui_component().size() +
+ std::vector<ui::AXNodeData> nodes(1 + // Root Node
visual_annotation.lines().size() +
paragraph_count + formatting_context_count);
size_t index = 0u;
- if (!visual_annotation.ui_component().empty()) {
- ui::AXNodeData& rootnode = nodes[index++];
- rootnode.role = ax::mojom::Role::kDialog;
- rootnode.id = GetNextNegativeNodeID();
- rootnode.relative_bounds.bounds = gfx::RectF(image_rect);
- for (const auto& ui_component : visual_annotation.ui_component())
- SerializeUIComponent(ui_component, index++, rootnode, nodes);
- }
+ // We assume that OCR is performed on a page-by-page basis.
+ ui::AXNodeData& page_node = nodes[index++];
+ page_node.role = ax::mojom::Role::kRegion;
+ page_node.id = GetNextNegativeNodeID();
+ update.root_id = page_node.id;
+ page_node.AddBoolAttribute(ax::mojom::BoolAttribute::kIsPageBreakingObject,
+ true);
+ page_node.relative_bounds.bounds = gfx::RectF(image_rect);
- if (!visual_annotation.lines().empty()) {
- // We assume that OCR is performed on a page-by-page basis.
- ui::AXNodeData& page_node = nodes[index++];
- page_node.role = ax::mojom::Role::kRegion;
- page_node.id = GetNextNegativeNodeID();
- update.root_id = page_node.id;
- page_node.AddBoolAttribute(ax::mojom::BoolAttribute::kIsPageBreakingObject,
- true);
- page_node.relative_bounds.bounds = gfx::RectF(image_rect);
+ // Add a disclaimer node informing the user of the beginning of extracted
+ // text, and place the message inside an appropriate ARIA landmark for easy
+ // navigation.
+ ui::AXNodeData& begin_node_wrapper = nodes[index++];
+ begin_node_wrapper.role = ax::mojom::Role::kBanner;
+ begin_node_wrapper.id = GetNextNegativeNodeID();
+ begin_node_wrapper.relative_bounds.bounds =
+ gfx::RectF(image_rect.x(), image_rect.y(), 1, 1);
+ page_node.child_ids.push_back(begin_node_wrapper.id);
+ ui::AXNodeData& begin_node = nodes[index++];
+ begin_node.role = ax::mojom::Role::kStaticText;
+ begin_node.id = GetNextNegativeNodeID();
+ begin_node.SetNameChecked(l10n_util::GetStringUTF8(IDS_PDF_OCR_RESULT_BEGIN));
+ begin_node.relative_bounds.bounds = begin_node_wrapper.relative_bounds.bounds;
+ begin_node_wrapper.child_ids.push_back(begin_node.id);
- // Add a disclaimer node informing the user of the beginning of extracted
- // text, and place the message inside an appropriate ARIA landmark for easy
- // navigation.
- ui::AXNodeData& begin_node_wrapper = nodes[index++];
- begin_node_wrapper.role = ax::mojom::Role::kBanner;
- begin_node_wrapper.id = GetNextNegativeNodeID();
- begin_node_wrapper.relative_bounds.bounds =
- gfx::RectF(image_rect.x(), image_rect.y(), 1, 1);
- page_node.child_ids.push_back(begin_node_wrapper.id);
- ui::AXNodeData& begin_node = nodes[index++];
- begin_node.role = ax::mojom::Role::kStaticText;
- begin_node.id = GetNextNegativeNodeID();
- begin_node.SetNameChecked(
- l10n_util::GetStringUTF8(IDS_PDF_OCR_RESULT_BEGIN));
- begin_node.relative_bounds.bounds =
- begin_node_wrapper.relative_bounds.bounds;
- begin_node_wrapper.child_ids.push_back(begin_node.id);
+ for (const auto& block_to_lines_pair : blocks_to_lines_map) {
+ // TODO(crbug.com/347622611): Create separate paragraphs based on the
+ // blocks' spacing (e.g. by utilizing heuristics found in
+ // PdfAccessibilityTree). Blocks as returned by the OCR engine are still
+ // too small.
+ ui::AXNodeData& paragraph_node = nodes[index++];
+ paragraph_node.role = ax::mojom::Role::kParagraph;
+ paragraph_node.id = GetNextNegativeNodeID();
+ page_node.child_ids.push_back(paragraph_node.id);
- for (const auto& block_to_lines_pair : blocks_to_lines_map) {
- // TODO(crbug.com/327298772): Create separate paragraphs based on the
- // blocks' spacing (e.g. by utilizing heuristics found in
- // PdfAccessibilityTree). Blocks as returned by the OCR engine are still
- // too small.
- ui::AXNodeData& paragraph_node = nodes[index++];
- paragraph_node.role = ax::mojom::Role::kParagraph;
- paragraph_node.id = GetNextNegativeNodeID();
- page_node.child_ids.push_back(paragraph_node.id);
+ for (const auto& line_sequence_number_to_index_pair :
+ block_to_lines_pair.second) {
+ const chrome_screen_ai::LineBox& line_box =
+ visual_annotation.lines(line_sequence_number_to_index_pair.second);
+ // Every line with a textual accessibility role should turn into one or
+ // more inline text boxes, each one representing a formatting context.
+ // If the line is not of a textual role, only one node is initialized
+ // having a more specific role such as `ax::mojom::Role::kImage`.
+ index += SerializeLineBox(line_box, index, paragraph_node, nodes);
- for (const auto& line_sequence_number_to_index_pair :
- block_to_lines_pair.second) {
- const chrome_screen_ai::LineBox& line_box =
- visual_annotation.lines(line_sequence_number_to_index_pair.second);
- // Every line with a textual accessibility role should turn into one or
- // more inline text boxes, each one representing a formatting context.
- // If the line is not of a textual role, only one node is initialized
- // having a more specific role such as `ax::mojom::Role::kImage`.
- index += SerializeLineBox(line_box, index, paragraph_node, nodes);
-
- // Accumulate bounds of all lines for the paragraph.
- auto& bounding_box = line_box.bounding_box();
- paragraph_node.relative_bounds.bounds.Union(
- gfx::RectF(bounding_box.x(), bounding_box.y(), bounding_box.width(),
- bounding_box.height()));
- }
+ // Accumulate bounds of all lines for the paragraph.
+ auto& bounding_box = line_box.bounding_box();
+ paragraph_node.relative_bounds.bounds.Union(
+ gfx::RectF(bounding_box.x(), bounding_box.y(), bounding_box.width(),
+ bounding_box.height()));
}
-
- // Add a disclaimer node informing the user of the end of extracted text,
- // and place the message inside an appropriate ARIA landmark for easy
- // navigation.
- ui::AXNodeData& end_node_wrapper = nodes[index++];
- end_node_wrapper.role = ax::mojom::Role::kContentInfo;
- end_node_wrapper.id = GetNextNegativeNodeID();
- end_node_wrapper.relative_bounds.bounds =
- gfx::RectF(image_rect.width(), image_rect.height(), 1, 1);
- page_node.child_ids.push_back(end_node_wrapper.id);
- ui::AXNodeData& end_node = nodes[index++];
- end_node.role = ax::mojom::Role::kStaticText;
- end_node.id = GetNextNegativeNodeID();
- end_node.SetNameChecked(l10n_util::GetStringUTF8(IDS_PDF_OCR_RESULT_END));
- end_node.relative_bounds.bounds = end_node_wrapper.relative_bounds.bounds;
- end_node_wrapper.child_ids.push_back(end_node.id);
}
+ // Add a disclaimer node informing the user of the end of extracted text,
+ // and place the message inside an appropriate ARIA landmark for easy
+ // navigation.
+ ui::AXNodeData& end_node_wrapper = nodes[index++];
+ end_node_wrapper.role = ax::mojom::Role::kContentInfo;
+ end_node_wrapper.id = GetNextNegativeNodeID();
+ end_node_wrapper.relative_bounds.bounds =
+ gfx::RectF(image_rect.width(), image_rect.height(), 1, 1);
+ page_node.child_ids.push_back(end_node_wrapper.id);
+ ui::AXNodeData& end_node = nodes[index++];
+ end_node.role = ax::mojom::Role::kStaticText;
+ end_node.id = GetNextNegativeNodeID();
+ end_node.SetNameChecked(l10n_util::GetStringUTF8(IDS_PDF_OCR_RESULT_END));
+ end_node.relative_bounds.bounds = end_node_wrapper.relative_bounds.bounds;
+ end_node_wrapper.child_ids.push_back(end_node.id);
+
// Filter out invalid / unrecognized / unused nodes from the update.
update.nodes.resize(nodes.size());
const auto end_node_iter = ranges::copy_if(
diff --git a/services/screen_ai/proto/visual_annotator_proto_convertor_unittest.cc b/services/screen_ai/proto/visual_annotator_proto_convertor_unittest.cc
index 00fb918..0faeb8d 100644
--- a/services/screen_ai/proto/visual_annotator_proto_convertor_unittest.cc
+++ b/services/screen_ai/proto/visual_annotator_proto_convertor_unittest.cc
@@ -84,51 +84,6 @@
using ScreenAIVisualAnnotatorProtoConvertorTest = testing::Test;
TEST_F(ScreenAIVisualAnnotatorProtoConvertorTest,
- VisualAnnotationToAXTreeUpdate_LayoutExtractionResults) {
- chrome_screen_ai::VisualAnnotation annotation;
- gfx::Rect snapshot_bounds(800, 900);
-
- screen_ai::ResetNodeIDForTesting();
-
- {
- chrome_screen_ai::UIComponent* component_0 = annotation.add_ui_component();
- chrome_screen_ai::UIComponent::PredictedType* type_0 =
- component_0->mutable_predicted_type();
- type_0->set_enum_type(chrome_screen_ai::UIComponent::BUTTON);
- chrome_screen_ai::Rect* box_0 = component_0->mutable_bounding_box();
- box_0->set_x(0);
- box_0->set_y(1);
- box_0->set_width(2);
- box_0->set_height(3);
- box_0->set_angle(90.0f);
-
- chrome_screen_ai::UIComponent* component_1 = annotation.add_ui_component();
- chrome_screen_ai::UIComponent::PredictedType* type_1 =
- component_1->mutable_predicted_type();
- type_1->set_string_type("Signature");
- chrome_screen_ai::Rect* box_1 = component_1->mutable_bounding_box();
- // `x`, `y`, and `angle` should be defaulted to 0 since they are singular
- // proto3 fields, not proto2.
- box_1->set_width(5);
- box_1->set_height(5);
- }
-
- {
- const ui::AXTreeUpdate update =
- VisualAnnotationToAXTreeUpdate(annotation, snapshot_bounds);
-
- const std::string expected_update(
- "id=-2 dialog child_ids=-3,-4 (0, 0)-(800, 900)\n"
- " id=-3 button (0, 1)-(2, 3)"
- " transform=[ 0 -1 0 0\n 1 0 0 0\n 0 0 1 0\n 0 0 0 1 ]\n"
- "\n"
- " id=-4 genericContainer (0, 0)-(5, 5) "
- "role_description=Signature\n");
- EXPECT_EQ(expected_update, update.ToString());
- }
-}
-
-TEST_F(ScreenAIVisualAnnotatorProtoConvertorTest,
VisualAnnotationToAXTreeUpdate_OcrResults) {
chrome_screen_ai::VisualAnnotation annotation;
gfx::Rect snapshot_bounds(800, 900);