blob: 035530a86f9ccd948781299b86841059d3e05734 [file] [log] [blame]
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
#include "base/macros.h"
#include "third_party/cld_3/src/src/nnet_language_identifier.h"
#include "ui/accessibility/ax_enums.mojom-forward.h"
#include "ui/accessibility/ax_export.h"
#include "ui/accessibility/ax_tree_observer.h"
namespace ui {
class AXNode;
class AXTree;
// This module implements language detection enabling Chrome to automatically
// detect the language for runs of text within the page.
// Node-level language detection runs once per page after the load complete
// event. This involves two passes:
// *Detect* walks the tree from the given root using cld3 to detect up to 3
// potential languages per node. A ranked list is created enumerating
// all potential languages on a page.
// *Label* re-walks the tree, assigning a language to each node considering
// the potential languages from the detect phase, page level
// statistics, and the assigned languages of ancestor nodes.
// Optionally an embedder may run *sub-node* language detection which attempts
// to assign languages for runs of text within a node, potentially down to the
// individual character level. This is useful in cases where a single paragraph
// involves switching between multiple languages, and where the speech engine
// doesn't automatically switch voices to handle different character sets.
// Due to the potentially small lengths of text runs involved this tends to be
// lower in accuracy, and works best when a node is composed of multiple
// languages with easily distinguishable scripts.
// AXLanguageInfo represents the local language detection data for all text
// within an AXNode. Stored on AXNode.
struct AX_EXPORT AXLanguageInfo {
// This is the final language we have assigned for this node during the
// 'label' step, it is the result of merging:
// a) The detected language for this node
// b) The declared lang attribute on this node
// c) the (recursive) language of the parent (detected or declared).
// This will be the empty string if no language was assigned during label
// phase.
// IETF BCP 47 Language code (rfc5646).
// examples:
// 'de'
// 'de-DE'
// 'en'
// 'en-US'
// 'es-ES'
// This should not be read directly by clients of AXNode, instead clients
// should call AXNode::GetLanguage().
// TODO(chrishall): consider renaming this to `assigned_language`.
std::string language;
// Detected languages for this node sorted as returned by
// FindTopNMostFreqLangs, which sorts in decreasing order of probability,
// filtered to remove any unreliable results.
std::vector<std::string> detected_languages;
// Each AXLanguageSpan contains a language, a probability, and start and end
// indices. The indices are used to specify the substring that contains the
// associated language. The string which the indices are relative to is not
// included in this structure.
// Also, the indices are relative to a Utf8 string.
// See documentation on GetLanguageAnnotationForStringAttribute for details
// on how to associate this object with a string.
struct AX_EXPORT AXLanguageSpan {
int start_index;
int end_index;
std::string language;
float probability;
// A single AXLanguageInfoStats instance is stored on each AXTree and contains
// statistics on detected languages for all the AXNodes in that tree.
// We rely on these tree-level statistics when labelling individual nodes, to
// provide extra signals to increase our confidence in assigning a detected
// language.
// These tree level statistics are also used to send reports on the language
// detection feature to enable tuning.
// The Label step will only assign a detected language to a node if that
// language is one of the most frequent languages on the page.
// For example, if a single node has detected_languages (in order of probability
// assigned by cld_3): da-DK, en-AU, fr-FR, but the page statistics overall
// indicate that the page is generally in en-AU and ja-JP, it is more likely to
// be a mis-recognition of Danish than an accurate assignment, so we assign
// en-AU instead of da-DK.
class AX_EXPORT AXLanguageInfoStats {
// Each AXLanguageInfoStats is tied to a specific AXTree, copying is safe but
// logically doesn't make sense.
AXLanguageInfoStats(const AXLanguageInfoStats&) = delete;
AXLanguageInfoStats& operator=(const AXLanguageInfoStats&) = delete;
// Adjust our statistics to add provided detected languages.
void Add(const std::vector<std::string>& languages);
// Fetch the score for a given language.
int GetScore(const std::string& lang) const;
// Check if a given language is within the top results.
bool CheckLanguageWithinTop(const std::string& lang);
// Record statistics based on how we labelled a node.
// We consider the language we labelled the node with, the language the author
// assigned, and whether or not we assigned our highest confidence detection
// result.
void RecordLabelStatistics(const std::string& labelled_lang,
const std::string& author_lang,
bool labelled_with_first_result);
// Update metrics to reflect we attempted to detect language for a node.
void RecordDetectionAttempt();
// Report metrics to UMA.
// Reports statistics since last run, run once detect & label iteration.
// If successful, will reset statistics.
void ReportMetrics();
// Allow access from a fixture only used in testing.
friend class AXLanguageDetectionTestFixture;
// Store a count of the occurrences of a given language.
std::unordered_map<std::string, int> lang_counts_;
// Cache of last calculated top language results.
// A vector of pairs of (score, language) sorted by descending score.
std::vector<std::pair<int, std::string>> top_results_;
// Boolean recording that we have not mutated the statistics since last
// calculating top results, setting this to false will cause recalculation
// when the results are next fetched.
bool top_results_valid_;
// Invalidate the top results cache.
void InvalidateTopResults();
// Compute the top results and store them in cache.
void GenerateTopResults();
// TODO(chrishall): Do we want this for testing? or is it better to only test
// the generated metrics by inspecting the histogram?
// Boolean used for testing metrics only, disables clearing of metrics.
bool disable_metric_clearing_;
void ClearMetrics();
// *** Statistics recorded for metric reporting. ***
// All statistics represent a single iteration of language detection and are
// reset after each successful call of ReportMetrics.
// The number of nodes we attempted detection on.
int count_detection_attempted_;
// The number of nodes we got detection results for.
int count_detection_results_;
// The number of nodes we assigned a label to.
int count_labelled_;
// The number of nodes we assigned a label to which was the highest confident
// detected language.
int count_labelled_with_top_result_;
// The number of times we labelled a language which disagreed with the node's
// author provided language annotation.
// If we have
// <div lang='en'><span>...</span><span>...</span></div>
// and we detect and label both spans as having language 'fr', then we count
// this as `2` overrides.
int count_overridden_;
// Set of top language detected for every node, used to generate the unique
// number of detected languages metric (LangsPerPage).
std::unordered_set<std::string> unique_top_lang_detected_;
// AXLanguageDetectionObserver is registered as a change observer on an AXTree
// and will run language detection after each update to the tree.
// We have kept this observer separate from the AXLanguageDetectionManager as we
// are aiming to launch language detection in two phases and wanted to try keep
// the code paths somewhat separate.
// TODO(chrishall): After both features have launched we could consider merging
// AXLanguageDetectionObserver into AXLanguageDetectionManager.
// TODO(chrishall): Investigate the cost of using AXTreeObserver, given that it
// has many empty virtual methods which are called for every AXTree change and
// we are only currently interested in OnAtomicUpdateFinished.
class AX_EXPORT AXLanguageDetectionObserver : public ui::AXTreeObserver {
// Observer constructor will register itself with the provided AXTree.
AXLanguageDetectionObserver(AXTree* tree);
// Observer destructor will remove itself as an observer from the AXTree.
~AXLanguageDetectionObserver() override;
// AXLanguageDetectionObserver contains a pointer so copying is non-trivial.
AXLanguageDetectionObserver(const AXLanguageDetectionObserver&) = delete;
AXLanguageDetectionObserver& operator=(const AXLanguageDetectionObserver&) =
void OnAtomicUpdateFinished(ui::AXTree* tree,
bool root_changed,
const std::vector<Change>& changes) override;
// Non-owning pointer to AXTree, used to de-register observer on destruction.
AXTree* const tree_;
// AXLanguageDetectionManager manages all of the context needed for language
// detection within an AXTree.
class AX_EXPORT AXLanguageDetectionManager {
// Construct an AXLanguageDetectionManager for the specified tree.
explicit AXLanguageDetectionManager(AXTree* tree);
// AXLanguageDetectionManager contains pointers so copying is non-trivial.
AXLanguageDetectionManager(const AXLanguageDetectionManager&) = delete;
AXLanguageDetectionManager& operator=(const AXLanguageDetectionManager&) =
// Detect languages for each node in the tree managed by this manager.
// This is the first pass in detection and labelling.
// This only detects the language, it does not label it, for that see
// LabelLanguageForSubtree.
void DetectLanguages();
// Label languages for each node in the tree manager by this manager.
// This is the second pass in detection and labelling.
// This will label the language, but relies on the earlier detection phase
// having already completed.
void LabelLanguages();
// Sub-node language detection for a given string attribute.
// For example, if a node has name: "My name is Fred", then calling
// GetLanguageAnnotationForStringAttribute(*node, ax::mojom::StringAttribute::
// kName) would return language detection information about "My name is Fred".
std::vector<AXLanguageSpan> GetLanguageAnnotationForStringAttribute(
const AXNode& node,
ax::mojom::StringAttribute attr);
// Construct and register a dynamic content change observer for this manager.
void RegisterLanguageDetectionObserver();
friend class AXLanguageDetectionObserver;
// Allow access from a fixture only used in testing.
friend class AXLanguageDetectionTestFixture;
// Helper methods to test if language detection features are enabled.
static bool IsStaticLanguageDetectionEnabled();
static bool IsDynamicLanguageDetectionEnabled();
// Perform detection for subtree rooted at subtree_root.
void DetectLanguagesForSubtree(AXNode* subtree_root);
// Perform detection for node. Will not descend into children.
void DetectLanguagesForNode(AXNode* node);
// Perform labelling for subtree rooted at subtree_root.
void LabelLanguagesForSubtree(AXNode* subtree_root);
// Perform labelling for node. Will not descend into children.
void LabelLanguagesForNode(AXNode* node);
// This language identifier is constructed with a default minimum byte length
// of chrome_lang_id::NNetLanguageIdentifier::kMinNumBytesToConsider and is
// used for detecting page-level languages.
chrome_lang_id::NNetLanguageIdentifier language_identifier_;
// This language identifier is constructed with a minimum byte length of
// kShortTextIdentifierMinByteLength so it can be used for detecting languages
// of shorter text (e.g. one character).
chrome_lang_id::NNetLanguageIdentifier short_text_language_identifier_;
// The observer to support dynamic content language detection.
std::unique_ptr<AXLanguageDetectionObserver> language_detection_observer_;
// Non-owning back pointer to the tree which owns this manager.
AXTree* tree_;
AXLanguageInfoStats lang_info_stats_;
} // namespace ui