[PDF OCR] Add a browser test for the function that generates a transform
Add a browser test for the function that generates a transform for
ui::AXRelativeBounds. This function was added to fix the bounding box
misalignment issue in crrev.com/c/4363403.
AX-Relnotes: n/a.
Bug: 1423810
Change-Id: I4e4a76675b9166ce77690eaa7d4e2641d63662f5
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4371269
Reviewed-by: David Tseng <dtseng@chromium.org>
Reviewed-by: Lei Zhang <thestig@chromium.org>
Commit-Queue: Kyungjun Lee <kyungjunlee@google.com>
Cr-Commit-Position: refs/heads/main@{#1126372}
diff --git a/components/BUILD.gn b/components/BUILD.gn
index 02d952e3..ba91007 100644
--- a/components/BUILD.gn
+++ b/components/BUILD.gn
@@ -911,6 +911,13 @@
]
}
+ if (enable_screen_ai_service) {
+ deps += [
+ "//components/services/screen_ai",
+ "//skia",
+ ]
+ }
+
if (is_android) {
sources += [
"browser_ui/client_certificate/android/ssl_client_certificate_request_browsertest.cc",
diff --git a/components/pdf/renderer/pdf_accessibility_tree.h b/components/pdf/renderer/pdf_accessibility_tree.h
index 2f854b62..678f50b 100644
--- a/components/pdf/renderer/pdf_accessibility_tree.h
+++ b/components/pdf/renderer/pdf_accessibility_tree.h
@@ -141,6 +141,8 @@
// request to the Screen AI library. The number of remaining OCR requests
// will decrement by one in `OnOcrDataReceived()`.
void IncrementNumberOfRemainingOcrRequests();
+
+ const ui::AXTree& tree_for_testing() const { return tree_; }
#endif // BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
bool ShowContextMenu();
diff --git a/components/pdf/renderer/pdf_accessibility_tree_browsertest.cc b/components/pdf/renderer/pdf_accessibility_tree_browsertest.cc
index 900a802..3115a0c 100644
--- a/components/pdf/renderer/pdf_accessibility_tree_browsertest.cc
+++ b/components/pdf/renderer/pdf_accessibility_tree_browsertest.cc
@@ -30,6 +30,15 @@
#include "ui/gfx/geometry/rect_conversions.h"
#include "ui/gfx/geometry/rect_f.h"
+#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+#include "components/services/screen_ai/screen_ai_ax_tree_serializer.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+#include "ui/accessibility/accessibility_features.h"
+#include "ui/accessibility/ax_node_data.h"
+#include "ui/accessibility/ax_tree_id.h"
+#include "ui/gfx/geometry/transform.h"
+#endif // BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+
namespace pdf {
namespace {
@@ -83,6 +92,37 @@
return (a << 24) | (r << 16) | (g << 8) | b;
}
+#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+ui::AXTreeUpdate CreateMockOCRResult(const gfx::RectF& image_bounds,
+ const gfx::RectF& text_bounds1,
+ const gfx::RectF& text_bounds2) {
+ ui::AXNodeData page_node;
+ page_node.role = ax::mojom::Role::kRegion;
+ page_node.id = 1001;
+ page_node.relative_bounds.bounds = image_bounds;
+
+ ui::AXNodeData text_node1;
+ text_node1.role = ax::mojom::Role::kStaticText;
+ text_node1.id = 1002;
+ text_node1.relative_bounds.bounds = text_bounds1;
+ page_node.child_ids.push_back(text_node1.id);
+
+ ui::AXNodeData text_node2;
+ text_node2.role = ax::mojom::Role::kStaticText;
+ text_node2.id = 1003;
+ text_node2.relative_bounds.bounds = text_bounds2;
+ page_node.child_ids.push_back(text_node2.id);
+
+ ui::AXTreeUpdate initial_state;
+ initial_state.root_id = page_node.id;
+ initial_state.nodes = {page_node, text_node1, text_node2};
+ initial_state.has_tree_data = true;
+ initial_state.tree_data.title = "OCR results";
+
+ return initial_state;
+}
+#endif // BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+
// This class overrides PdfAccessibilityActionHandler to record received
// action data when tests make an accessibility action call.
class TestPdfAccessibilityActionHandler
@@ -2102,4 +2142,190 @@
}
}
+#if BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+TEST_F(PdfAccessibilityTreeTest, TestTransformFromOnOcrDataReceived) {
+ // Enable feature flag.
+ base::test::ScopedFeatureList scoped_feature_list;
+ scoped_feature_list.InitAndEnableFeature(::features::kPdfOcr);
+
+ // Assume `image` contains some text that will be extracted by OCR. `image`
+ // will be passed to the function that creates a transform, which will be
+ // then applied to the text paragraphs extracted by OCR.
+ chrome_pdf::AccessibilityImageInfo image;
+ constexpr float kImageWidth = 200.0f;
+ constexpr float kImageHeight = 200.0f;
+ image.bounds = gfx::RectF(0.0f, 0.0f, kImageWidth, kImageHeight);
+ // Simulate that the width and height of `image` got shrunk by 80% in
+ // `image_data`.
+ constexpr float kScaleFactor = 0.8f;
+ image.image_data.allocN32Pixels(static_cast<int>(kImageWidth * kScaleFactor),
+ static_cast<int>(kImageHeight * kScaleFactor),
+ /*isOpaque=*/false);
+ page_objects_.images.push_back(image);
+
+ page_info_.text_run_count = text_runs_.size();
+ page_info_.char_count = chars_.size();
+
+ content::RenderFrame* render_frame = GetMainRenderFrame();
+ ASSERT_TRUE(render_frame);
+ render_frame->SetAccessibilityModeForTest(ui::AXMode::kWebContents);
+ ASSERT_TRUE(render_frame->GetRenderAccessibility());
+
+ TestPdfAccessibilityActionHandler action_handler;
+ PdfAccessibilityTree pdf_accessibility_tree(render_frame, &action_handler);
+
+ pdf_accessibility_tree.SetAccessibilityViewportInfo(viewport_info_);
+ pdf_accessibility_tree.SetAccessibilityDocInfo(doc_info_);
+ pdf_accessibility_tree.SetAccessibilityPageInfo(page_info_, text_runs_,
+ chars_, page_objects_);
+ WaitForThreadTasks();
+
+ // TODO(crbug.com/1423810): Convert these in-line comments into EXPECT() with
+ // ToString() output from AXTree. To do this in a more stable way, we need to
+ // use AXTreeFormatter and move the whole or some part of the following file
+ // (content/public/browser/ax_inspect_factory.h) into content/public/renderer
+ // or content/public/common along with its deps.
+ /*
+ * Expected PDF accessibility tree structure (with PDF OCR feature flag)
+ * Document
+ * ++ Status
+ * ++ Region
+ * ++++ Paragraph
+ * ++++++ image
+ */
+
+ const ui::AXTree& ax_tree_in_pdf = pdf_accessibility_tree.tree_for_testing();
+ ui::AXNode* root_node = ax_tree_in_pdf.root();
+ ASSERT_TRUE(root_node);
+ EXPECT_EQ(ax::mojom::Role::kPdfRoot, root_node->GetRole());
+ ASSERT_EQ(2u, root_node->children().size());
+
+ ui::AXNode* status_node = root_node->children()[0];
+ ASSERT_TRUE(status_node);
+ EXPECT_EQ(ax::mojom::Role::kStatus, status_node->GetRole());
+ ASSERT_EQ(0u, status_node->children().size());
+
+ ui::AXNode* page_node = root_node->children()[1];
+ ASSERT_TRUE(page_node);
+ EXPECT_EQ(ax::mojom::Role::kRegion, page_node->GetRole());
+ ASSERT_EQ(1u, page_node->children().size());
+
+ ui::AXNode* paragraph_node = page_node->children()[0];
+ ASSERT_TRUE(paragraph_node);
+ EXPECT_EQ(ax::mojom::Role::kParagraph, paragraph_node->GetRole());
+ ASSERT_EQ(1u, paragraph_node->children().size());
+
+ ui::AXNode* image_node = paragraph_node->children()[0];
+ ASSERT_TRUE(image_node);
+ EXPECT_EQ(ax::mojom::Role::kImage, image_node->GetRole());
+ ASSERT_EQ(0u, image_node->children().size());
+ EXPECT_EQ(image.bounds, image_node->data().relative_bounds.bounds);
+
+ // Simulate creating a child tree using OCR results.
+ pdf_accessibility_tree.IncrementNumberOfRemainingOcrRequests();
+ // Text bounds before applying the transform.
+ constexpr gfx::RectF kTextBoundsBeforeTransform1 = {{8.0f, 8.0f},
+ {80.0f, 24.0f}};
+ constexpr gfx::RectF kTextBoundsBeforeTransform2 = {{16.0f, 88.0f},
+ {40.0f, 56.0f}};
+ ui::AXTreeUpdate initial_state = CreateMockOCRResult(
+ image.bounds, kTextBoundsBeforeTransform1, kTextBoundsBeforeTransform2);
+ screen_ai::ScreenAIAXTreeSerializer serializer(
+ render_frame->GetRenderAccessibility()->GetTreeIDForPluginHost(),
+ std::move(initial_state.nodes));
+ WaitForThreadTasks();
+
+ const ui::AXTree* child_tree = serializer.tree_for_testing();
+ ASSERT_TRUE(child_tree);
+ EXPECT_NE(child_tree->GetAXTreeID(), ui::AXTreeIDUnknown());
+ EXPECT_NE(child_tree->GetAXTreeID(), ax_tree_in_pdf.GetAXTreeID());
+ pdf_accessibility_tree.OnOcrDataReceived(
+ image_node->id(), image, paragraph_node->id(), child_tree->GetAXTreeID());
+ WaitForThreadTasks();
+
+ // TODO(crbug.com/1423810): Convert these in-line comments into EXPECT() with
+ // ToString() output from AXTree. To do this in a more stable way, we need to
+ // use AXTreeFormatter and move the whole or some part of the following file
+ // (content/public/browser/ax_inspect_factory.h) into content/public/renderer
+ // or content/public/common along with its deps.
+ /*
+ * Expected PDF accessibility tree structure (after running OCR)
+ * Document
+ * ++ Status
+ * ++ Region
+ * ++++ Paragraph
+ * ++++++ Region (child tree)
+ * ++++++++ Static Text
+ * ++++++++ Static Text
+ */
+
+ root_node = ax_tree_in_pdf.root();
+ ASSERT_TRUE(root_node);
+ ASSERT_EQ(2u, root_node->children().size());
+
+ status_node = root_node->children()[0];
+ ASSERT_TRUE(status_node);
+ EXPECT_EQ(ax::mojom::Role::kStatus, status_node->GetRole());
+
+ page_node = root_node->children()[1];
+ ASSERT_TRUE(page_node);
+ EXPECT_EQ(ax::mojom::Role::kRegion, page_node->GetRole());
+ ASSERT_EQ(1u, page_node->children().size());
+
+ paragraph_node = page_node->children()[0];
+ ASSERT_TRUE(paragraph_node);
+ EXPECT_EQ(ax::mojom::Role::kParagraph, paragraph_node->GetRole());
+ ASSERT_EQ(1u, paragraph_node->children().size());
+
+ ui::AXNode* host_node = paragraph_node->children()[0];
+ ASSERT_TRUE(host_node);
+ EXPECT_TRUE(
+ host_node->HasStringAttribute(ax::mojom::StringAttribute::kChildTreeId));
+ ui::AXTreeID child_tree_id = ui::AXTreeID::FromString(
+ host_node->GetStringAttribute(ax::mojom::StringAttribute::kChildTreeId));
+ ASSERT_EQ(child_tree_id, child_tree->GetAXTreeID());
+
+ ASSERT_TRUE(host_node->data().relative_bounds.transform.get());
+
+ // Expected text bounds after applying the transform. These numbers are
+ // expected to be kTextBoundsBeforeTransform * 1 / kScaleFactor.
+ constexpr gfx::RectF kExpectedTextBoundRelativeToTreeBounds1 = {
+ {10.0f, 10.0f}, {100.0f, 30.0f}};
+ constexpr gfx::RectF kExpectedTextBoundRelativeToTreeBounds2 = {
+ {20.0f, 110.0f}, {50.0f, 70.0f}};
+
+ // Check the child tree.
+ ui::AXNode* child_tree_host_node = child_tree->root();
+ ASSERT_TRUE(child_tree_host_node);
+ EXPECT_EQ(ax::mojom::Role::kRegion, child_tree_host_node->GetRole());
+ ASSERT_EQ(2u, child_tree_host_node->children().size());
+
+ ui::AXNode* child_tree_node = child_tree_host_node->children()[0];
+ ASSERT_TRUE(child_tree_node);
+ EXPECT_EQ(ax::mojom::Role::kStaticText, child_tree_node->GetRole());
+ gfx::RectF bounds = child_tree_node->data().relative_bounds.bounds;
+ CompareRect(kTextBoundsBeforeTransform1, bounds);
+ // After applying the transform via RelativeToTreeBounds.
+ bounds = child_tree->RelativeToTreeBounds(child_tree_node, bounds,
+ /*offscreen=*/nullptr,
+ /*clip_bounds=*/true,
+ /*skip_container_offset=*/true);
+ bounds = ax_tree_in_pdf.RelativeToTreeBounds(host_node, bounds);
+ CompareRect(kExpectedTextBoundRelativeToTreeBounds1, bounds);
+
+ child_tree_node = child_tree_host_node->children()[1];
+ ASSERT_TRUE(child_tree_node);
+ EXPECT_EQ(ax::mojom::Role::kStaticText, child_tree_node->GetRole());
+ bounds = child_tree_node->data().relative_bounds.bounds;
+ CompareRect(kTextBoundsBeforeTransform2, bounds);
+ // After applying the transform via RelativeToTreeBounds.
+ bounds = child_tree->RelativeToTreeBounds(child_tree_node, bounds,
+ /*offscreen=*/nullptr,
+ /*clip_bounds=*/true,
+ /*skip_container_offset=*/true);
+ bounds = ax_tree_in_pdf.RelativeToTreeBounds(host_node, bounds);
+ CompareRect(kExpectedTextBoundRelativeToTreeBounds2, bounds);
+}
+#endif // BUILDFLAG(ENABLE_SCREEN_AI_SERVICE)
+
} // namespace pdf
diff --git a/components/services/screen_ai/screen_ai_ax_tree_serializer.h b/components/services/screen_ai/screen_ai_ax_tree_serializer.h
index 1d781d7..e5226988 100644
--- a/components/services/screen_ai/screen_ai_ax_tree_serializer.h
+++ b/components/services/screen_ai/screen_ai_ax_tree_serializer.h
@@ -36,6 +36,8 @@
ui::AXTreeUpdate Serialize() const;
+ const ui::AXTree* tree_for_testing() const { return tree_.get(); }
+
private:
const std::unique_ptr<ui::AXSerializableTree> tree_;
std::unique_ptr<ui::AXTreeSource<const ui::AXNode*>> tree_source_;
