blob: eea64e9fa41aab0579005611f19fc0dacaacc817 [file] [log] [blame]
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "ui/accessibility/ax_language_detection.h"
#include <algorithm>
#include <functional>
#include "base/command_line.h"
#include "base/i18n/unicodestring.h"
#include "base/strings/utf_string_conversions.h"
#include "base/trace_event/trace_event.h"
#include "ui/accessibility/accessibility_switches.h"
#include "ui/accessibility/ax_enums.mojom.h"
#include "ui/accessibility/ax_tree.h"
namespace ui {
namespace {
// This is the maximum number of languages we assign per page, so only the top
// 3 languages on the top will be assigned to any node.
const auto kMaxDetectedLanguagesPerPage = 3;
// This is the maximum number of languages that cld3 will detect for each
// input we give it, 3 was recommended to us by the ML team as a good
// starting point.
const auto kMaxDetectedLanguagesPerSpan = 3;
const auto kShortTextIdentifierMinByteLength = 1;
// TODO(https://bugs.chromium.org/p/chromium/issues/detail?id=971360):
// Determine appropriate value for kShortTextIdentifierMaxByteLength.
const auto kShortTextIdentifierMaxByteLength = 1000;
} // namespace
AXLanguageInfo::AXLanguageInfo() = default;
AXLanguageInfo::~AXLanguageInfo() = default;
AXLanguageInfoStats::AXLanguageInfoStats() : top_results_valid_(false) {}
AXLanguageInfoStats::~AXLanguageInfoStats() = default;
void AXLanguageInfoStats::Add(const std::vector<std::string>& languages) {
// Assign languages with higher probability a higher score.
// TODO(chrishall): consider more complex scoring
size_t score = kMaxDetectedLanguagesPerSpan;
for (const auto& lang : languages) {
lang_counts_[lang] += score;
--score;
}
InvalidateTopResults();
}
int AXLanguageInfoStats::GetScore(const std::string& lang) const {
const auto& lang_count_it = lang_counts_.find(lang);
if (lang_count_it == lang_counts_.end()) {
return 0;
}
return lang_count_it->second;
}
void AXLanguageInfoStats::InvalidateTopResults() {
top_results_valid_ = false;
}
// Check if a given language is within the top results.
bool AXLanguageInfoStats::CheckLanguageWithinTop(const std::string& lang) {
if (!top_results_valid_) {
GenerateTopResults();
}
for (const auto& item : top_results_) {
if (lang == item.second) {
return true;
}
}
return false;
}
void AXLanguageInfoStats::GenerateTopResults() {
top_results_.clear();
for (const auto& item : lang_counts_) {
top_results_.emplace_back(item.second, item.first);
}
// Since we store the pair as (score, language) the default operator> on pairs
// does our sort appropriately.
// Sort in descending order.
std::sort(top_results_.begin(), top_results_.end(),
std::greater<std::pair<unsigned int, std::string>>());
// Resize down to remove all values greater than the N we are considering.
top_results_.resize(kMaxDetectedLanguagesPerPage);
top_results_valid_ = true;
}
AXLanguageDetectionManager::AXLanguageDetectionManager(AXTree* tree)
: short_text_language_identifier_(kShortTextIdentifierMinByteLength,
kShortTextIdentifierMaxByteLength),
tree_(tree) {}
AXLanguageDetectionManager::~AXLanguageDetectionManager() = default;
void AXLanguageDetectionManager::RegisterLanguageDetectionObserver() {
// If the dynamic feature flag is not enabled then do nothing.
if (!::switches::
IsExperimentalAccessibilityLanguageDetectionDynamicEnabled()) {
return;
}
// Construct our new Observer as requested.
// If there is already an Observer on this Manager then this will destroy it.
language_detection_observer_.reset(new AXLanguageDetectionObserver(tree_));
}
// Detect languages for each node.
void AXLanguageDetectionManager::DetectLanguages() {
TRACE_EVENT0("accessibility", "AXLanguageInfo::DetectLanguages");
if (!::switches::IsExperimentalAccessibilityLanguageDetectionEnabled()) {
return;
}
DetectLanguagesForSubtree(tree_->root());
}
// Detect languages for a subtree rooted at the given subtree_root.
// Will not check feature flag.
void AXLanguageDetectionManager::DetectLanguagesForSubtree(
AXNode* subtree_root) {
// Only perform detection for kStaticText(s).
//
// Do not visit the children of kStaticText(s) as they don't have
// interesting children for language detection.
//
// Since kInlineTextBox(es) contain text from their parent, any detection on
// them is redundant. Instead they can inherit the detected language.
if (subtree_root->data().role == ax::mojom::Role::kStaticText) {
DetectLanguagesForNode(subtree_root);
} else {
// Otherwise, recurse into children for detection.
for (AXNode* child : subtree_root->children()) {
DetectLanguagesForSubtree(child);
}
}
}
// Detect languages for a single node.
// Will not descend into children.
// Will not check feature flag.
void AXLanguageDetectionManager::DetectLanguagesForNode(AXNode* node) {
// TODO(chrishall): implement strategy for nodes which are too small to get
// reliable language detection results. Consider combination of
// concatenation and bubbling up results.
auto text = node->GetStringAttribute(ax::mojom::StringAttribute::kName);
// FindTopNMostFreqLangs will pad the results with
// NNetLanguageIdentifier::kUnknown in order to reach the requested number
// of languages, this means we cannot rely on the results' length and we
// have to filter the results.
const auto results = language_identifier_.FindTopNMostFreqLangs(
text, kMaxDetectedLanguagesPerSpan);
std::vector<std::string> reliable_results;
for (const auto res : results) {
// The output of FindTopNMostFreqLangs is already sorted by byte count,
// this seems good enough for now.
// Only consider results which are 'reliable', this will also remove
// 'unknown'.
if (res.is_reliable) {
reliable_results.push_back(res.language);
}
}
// Only allocate a(n) LanguageInfo if we have results worth keeping.
if (reliable_results.size()) {
AXLanguageInfo* lang_info = node->GetLanguageInfo();
if (lang_info) {
// Clear previously detected and labelled languages.
lang_info->detected_languages.clear();
lang_info->language.clear();
} else {
node->SetLanguageInfo(std::make_unique<AXLanguageInfo>());
lang_info = node->GetLanguageInfo();
}
// Keep these results.
lang_info->detected_languages = std::move(reliable_results);
// Update statistics to take these results into account.
lang_info_stats_.Add(lang_info->detected_languages);
}
}
// Label languages for each node. This relies on DetectLanguages having already
// been run.
void AXLanguageDetectionManager::LabelLanguages() {
TRACE_EVENT0("accessibility", "AXLanguageInfo::LabelLanguages");
if (!::switches::IsExperimentalAccessibilityLanguageDetectionEnabled()) {
return;
}
LabelLanguagesForSubtree(tree_->root());
}
// Label languages for each node in the subtree rooted at the given
// subtree_root. Will not check feature flag.
void AXLanguageDetectionManager::LabelLanguagesForSubtree(
AXNode* subtree_root) {
LabelLanguagesForNode(subtree_root);
// Recurse into children to continue labelling.
for (AXNode* child : subtree_root->children()) {
LabelLanguagesForSubtree(child);
}
}
// Label languages for a single node.
// Will not descend into children.
// Will not check feature flag.
void AXLanguageDetectionManager::LabelLanguagesForNode(AXNode* node) {
AXLanguageInfo* lang_info = node->GetLanguageInfo();
// lang_info is only attached by Detect when it thinks a node is interesting,
// the presence of lang_info means that Detect expects the node to end up with
// a language specified.
//
// If the lang_info->language is already set then we have no more work to do
// for this node.
if (lang_info && lang_info->language.empty()) {
// We assign the highest probability language which is both:
// 1) reliably detected for this node, and
// 2) one of the top (kMaxDetectedLanguagesPerPage) languages on this page.
//
// This helps guard against false positives for nodes which have noisy
// language detection results in isolation.
for (const auto& lang : lang_info->detected_languages) {
if (lang_info_stats_.CheckLanguageWithinTop(lang)) {
lang_info->language = lang;
break;
}
}
// After attempting labelling we no longer need the detected results in
// LanguageInfo, as they have no future use.
if (lang_info->language.empty()) {
// If no language was assigned then LanguageInfo as a whole can safely be
// destroyed.
node->ClearLanguageInfo();
} else {
// Otherwise, if we assigned a language then we need to keep
// LanguageInfo.language, but we can clear the detected results.
lang_info->detected_languages.clear();
}
}
}
std::vector<AXLanguageSpan>
AXLanguageDetectionManager::GetLanguageAnnotationForStringAttribute(
const AXNode& node,
ax::mojom::StringAttribute attr) {
std::vector<AXLanguageSpan> language_annotation;
if (!node.HasStringAttribute(attr))
return language_annotation;
std::string attr_value = node.GetStringAttribute(attr);
// Use author-provided language if present.
if (node.HasStringAttribute(ax::mojom::StringAttribute::kLanguage)) {
// Use author-provided language if present.
language_annotation.push_back(AXLanguageSpan{
0 /* start_index */, attr_value.length() /* end_index */,
node.GetStringAttribute(
ax::mojom::StringAttribute::kLanguage) /* language */,
1 /* probability */});
return language_annotation;
}
// Calculate top 3 languages.
// TODO(akihiroota): What's a reasonable number of languages to have
// cld_3 find? Should vary.
std::vector<chrome_lang_id::NNetLanguageIdentifier::Result> top_languages =
short_text_language_identifier_.FindTopNMostFreqLangs(
attr_value, kMaxDetectedLanguagesPerPage);
// Create vector of AXLanguageSpans.
for (const auto& result : top_languages) {
std::vector<chrome_lang_id::NNetLanguageIdentifier::SpanInfo> ranges =
result.byte_ranges;
for (const auto& span_info : ranges) {
language_annotation.push_back(
AXLanguageSpan{span_info.start_index, span_info.end_index,
result.language, span_info.probability});
}
}
// Sort Language Annotations by increasing start index. LanguageAnnotations
// with lower start index should appear earlier in the vector.
std::sort(
language_annotation.begin(), language_annotation.end(),
[](const AXLanguageSpan& left, const AXLanguageSpan& right) -> bool {
return left.start_index <= right.start_index;
});
// Ensure that AXLanguageSpans do not overlap.
for (size_t i = 0; i < language_annotation.size(); ++i) {
if (i > 0) {
DCHECK(language_annotation[i].start_index <=
language_annotation[i - 1].end_index);
}
}
return language_annotation;
}
AXLanguageDetectionObserver::AXLanguageDetectionObserver(AXTree* tree)
: tree_(tree) {
// We expect the feature flag to have be checked before this Observer is
// constructed, this should have been checked by
// RegisterLanguageDetectionObserver.
DCHECK(
::switches::IsExperimentalAccessibilityLanguageDetectionDynamicEnabled());
tree_->AddObserver(this);
}
AXLanguageDetectionObserver::~AXLanguageDetectionObserver() {
tree_->RemoveObserver(this);
}
void AXLanguageDetectionObserver::OnAtomicUpdateFinished(
ui::AXTree* tree,
bool root_changed,
const std::vector<Change>& changes) {
// TODO(chrishall): We likely want to re-consider updating or resetting
// AXLanguageInfoStats over time to better support detection on long running
// pages.
// TODO(chrishall): To support pruning deleted node data from stats we should
// consider implementing OnNodeWillBeDeleted. Other options available include:
// 1) move lang info from AXNode into a map on AXTree so that we can fetch
// based on id in here
// 2) AXLanguageInfo destructor could remove itself
// TODO(chrishall): Possible optimisation: only run detect/label for certain
// change.type(s)), at least NODE_CREATED, NODE_CHANGED, and SUBTREE_CREATED.
DCHECK(tree->language_detection_manager);
// Perform Detect and Label for each node changed or created.
// We currently only consider kStaticText for detection.
//
// Note that language inheritance is now handled by AXNode::GetLanguage.
//
// Note that since Label no longer handles language inheritance, we only need
// to call Label and Detect on the nodes that changed and don't need to
// recurse.
//
// We do this in two passes because Detect updates page level statistics which
// are later used by Label in order to make more accurate decisions.
for (auto& change : changes) {
if (change.node->data().role == ax::mojom::Role::kStaticText) {
tree->language_detection_manager->DetectLanguagesForNode(change.node);
}
}
for (auto& change : changes) {
if (change.node->data().role == ax::mojom::Role::kStaticText) {
tree->language_detection_manager->LabelLanguagesForNode(change.node);
}
}
}
} // namespace ui