blob: 213d7d340fa2dabff75295c6ec347b36511ba940 [file] [log] [blame]
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/autofill/core/browser/autofill_metrics.h"
#include <stddef.h>
#include <memory>
#include <utility>
#include <vector>
#include "base/feature_list.h"
#include "base/macros.h"
#include "base/metrics/metrics_hashes.h"
#include "base/metrics/statistics_recorder.h"
#include "base/strings/string16.h"
#include "base/strings/utf_string_conversions.h"
#include "base/test/metrics/histogram_tester.h"
#include "base/test/metrics/user_action_tester.h"
#include "base/test/scoped_feature_list.h"
#include "base/test/scoped_task_environment.h"
#include "base/time/time.h"
#include "components/autofill/core/browser/autofill_external_delegate.h"
#include "components/autofill/core/browser/autofill_test_utils.h"
#include "components/autofill/core/browser/mock_autocomplete_history_manager.h"
#include "components/autofill/core/browser/payments/test_payments_client.h"
#include "components/autofill/core/browser/personal_data_manager.h"
#include "components/autofill/core/browser/popup_item_ids.h"
#include "components/autofill/core/browser/sync_utils.h"
#include "components/autofill/core/browser/test_autofill_client.h"
#include "components/autofill/core/browser/test_autofill_driver.h"
#include "components/autofill/core/browser/test_autofill_manager.h"
#include "components/autofill/core/browser/test_credit_card_save_manager.h"
#include "components/autofill/core/browser/test_form_data_importer.h"
#include "components/autofill/core/browser/test_form_structure.h"
#include "components/autofill/core/browser/test_personal_data_manager.h"
#include "components/autofill/core/browser/webdata/autofill_webdata_service.h"
#include "components/autofill/core/common/autofill_clock.h"
#include "components/autofill/core/common/autofill_features.h"
#include "components/autofill/core/common/form_data.h"
#include "components/autofill/core/common/form_field_data.h"
#include "components/prefs/pref_service.h"
#include "components/sync/driver/test_sync_service.h"
#include "components/ukm/test_ukm_recorder.h"
#include "components/webdata/common/web_data_results.h"
#include "services/metrics/public/cpp/ukm_builders.h"
#include "services/metrics/public/cpp/ukm_source.h"
#include "services/network/public/cpp/shared_url_loader_factory.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "ui/gfx/geometry/rect.h"
#include "url/gurl.h"
using autofill::features::kAutofillEnforceMinRequiredFieldsForHeuristics;
using autofill::features::kAutofillEnforceMinRequiredFieldsForQuery;
using autofill::features::kAutofillEnforceMinRequiredFieldsForUpload;
using SyncSigninState = autofill::AutofillSyncSigninState;
using base::ASCIIToUTF16;
using base::Bucket;
using base::TimeTicks;
using ::testing::ElementsAre;
using ::testing::Matcher;
using ::testing::UnorderedPointwise;
namespace autofill {
namespace {
using UkmCardUploadDecisionType = ukm::builders::Autofill_CardUploadDecision;
using UkmDeveloperEngagementType = ukm::builders::Autofill_DeveloperEngagement;
using UkmInteractedWithFormType = ukm::builders::Autofill_InteractedWithForm;
using UkmSuggestionsShownType = ukm::builders::Autofill_SuggestionsShown;
using UkmSelectedMaskedServerCardType =
ukm::builders::Autofill_SelectedMaskedServerCard;
using UkmSuggestionFilledType = ukm::builders::Autofill_SuggestionFilled;
using UkmTextFieldDidChangeType = ukm::builders::Autofill_TextFieldDidChange;
using UkmLogHiddenRepresentationalFieldSkipDecisionType =
ukm::builders::Autofill_HiddenRepresentationalFieldSkipDecision;
using UkmLogRepeatedServerTypePredictionRationalized =
ukm::builders::Autofill_RepeatedServerTypePredictionRationalized;
using UkmFormSubmittedType = ukm::builders::Autofill_FormSubmitted;
using UkmFieldTypeValidationType = ukm::builders::Autofill_FieldTypeValidation;
using UkmFieldFillStatusType = ukm::builders::Autofill_FieldFillStatus;
using ExpectedUkmMetrics =
std::vector<std::vector<std::pair<const char*, int64_t>>>;
int64_t Collapse(uint64_t sig) {
return sig % 1021;
}
void VerifyDeveloperEngagementUkm(
const ukm::TestAutoSetUkmRecorder& ukm_recorder,
const FormData& form,
const bool is_for_credit_card,
const std::set<FormType>& form_types,
const std::vector<int64_t>& expected_metric_values) {
int expected_metric_value = 0;
for (const auto it : expected_metric_values)
expected_metric_value |= 1 << it;
auto entries =
ukm_recorder.GetEntriesByName(UkmDeveloperEngagementType::kEntryName);
EXPECT_EQ(1u, entries.size());
for (const auto* const entry : entries) {
ukm_recorder.ExpectEntrySourceHasUrl(entry,
GURL(form.main_frame_origin.GetURL()));
EXPECT_EQ(4u, entry->metrics.size());
ukm_recorder.ExpectEntryMetric(
entry, UkmDeveloperEngagementType::kDeveloperEngagementName,
expected_metric_value);
ukm_recorder.ExpectEntryMetric(
entry, UkmDeveloperEngagementType::kIsForCreditCardName,
is_for_credit_card);
ukm_recorder.ExpectEntryMetric(
entry, UkmDeveloperEngagementType::kFormTypesName,
AutofillMetrics::FormTypesToBitVector(form_types));
ukm_recorder.ExpectEntryMetric(
entry, UkmDeveloperEngagementType::kFormSignatureName,
Collapse(CalculateFormSignature(form)));
}
}
MATCHER(CompareMetricsIgnoringMillisecondsSinceFormParsed, "") {
const auto& lhs = ::testing::get<0>(arg);
const std::pair<const char*, int64_t>& rhs = ::testing::get<1>(arg);
return lhs.first == base::HashMetricName(rhs.first) &&
(lhs.second == rhs.second ||
(lhs.second > 0 &&
rhs.first ==
UkmSuggestionFilledType::kMillisecondsSinceFormParsedName));
}
void VerifyFormInteractionUkm(const ukm::TestAutoSetUkmRecorder& ukm_recorder,
const FormData& form,
const char* event_name,
const ExpectedUkmMetrics& expected_metrics) {
auto entries = ukm_recorder.GetEntriesByName(event_name);
EXPECT_LE(entries.size(), expected_metrics.size());
for (size_t i = 0; i < expected_metrics.size() && i < entries.size(); i++) {
ukm_recorder.ExpectEntrySourceHasUrl(entries[i],
GURL(form.main_frame_origin.GetURL()));
EXPECT_THAT(
entries[i]->metrics,
UnorderedPointwise(CompareMetricsIgnoringMillisecondsSinceFormParsed(),
expected_metrics[i]));
}
}
void VerifySubmitFormUkm(const ukm::TestAutoSetUkmRecorder& ukm_recorder,
const FormData& form,
AutofillMetrics::AutofillFormSubmittedState state,
bool is_for_credit_card,
bool has_upi_vpa_field,
const std::set<FormType>& form_types) {
VerifyFormInteractionUkm(
ukm_recorder, form, UkmFormSubmittedType::kEntryName,
{{{UkmFormSubmittedType::kAutofillFormSubmittedStateName, state},
{UkmSuggestionFilledType::kMillisecondsSinceFormParsedName, 0},
{UkmFormSubmittedType::kIsForCreditCardName, is_for_credit_card},
{UkmFormSubmittedType::kHasUpiVpaFieldName, has_upi_vpa_field},
{UkmFormSubmittedType::kFormTypesName,
AutofillMetrics::FormTypesToBitVector(form_types)},
{UkmFormSubmittedType::kFormSignatureName,
Collapse(CalculateFormSignature(form))}}});
}
void AppendFieldFillStatusUkm(const FormData& form,
ExpectedUkmMetrics* expected_metrics) {
int64_t form_signature = Collapse(CalculateFormSignature(form));
int64_t metric_type = static_cast<int64_t>(AutofillMetrics::TYPE_SUBMISSION);
for (const FormFieldData& field : form.fields) {
int64_t field_signature = Collapse(CalculateFieldSignatureForField(field));
expected_metrics->push_back(
{{UkmSuggestionFilledType::kMillisecondsSinceFormParsedName, 0},
{UkmFieldFillStatusType::kFormSignatureName, form_signature},
{UkmFieldFillStatusType::kFieldSignatureName, field_signature},
{UkmFieldFillStatusType::kValidationEventName, metric_type},
{UkmTextFieldDidChangeType::kIsAutofilledName,
field.is_autofilled ? 1 : 0},
{UkmFieldFillStatusType::kWasPreviouslyAutofilledName, 0}});
}
}
void AppendFieldTypeUkm(const FormData& form,
const std::vector<ServerFieldType>& heuristic_types,
const std::vector<ServerFieldType>& server_types,
const std::vector<ServerFieldType>& actual_types,
ExpectedUkmMetrics* expected_metrics) {
ASSERT_EQ(heuristic_types.size(), form.fields.size());
ASSERT_EQ(server_types.size(), form.fields.size());
ASSERT_EQ(actual_types.size(), form.fields.size());
int64_t form_signature = Collapse(CalculateFormSignature(form));
int64_t metric_type = static_cast<int64_t>(AutofillMetrics::TYPE_SUBMISSION);
std::vector<int64_t> prediction_sources{
AutofillMetrics::PREDICTION_SOURCE_HEURISTIC,
AutofillMetrics::PREDICTION_SOURCE_SERVER,
AutofillMetrics::PREDICTION_SOURCE_OVERALL};
for (size_t i = 0; i < form.fields.size(); ++i) {
const FormFieldData& field = form.fields[i];
int64_t field_signature = Collapse(CalculateFieldSignatureForField(field));
for (int64_t source : prediction_sources) {
int64_t predicted_type = static_cast<int64_t>(
(source == AutofillMetrics::PREDICTION_SOURCE_SERVER
? server_types
: heuristic_types)[i]);
int64_t actual_type = static_cast<int64_t>(actual_types[i]);
expected_metrics->push_back(
{{UkmSuggestionFilledType::kMillisecondsSinceFormParsedName, 0},
{UkmFieldFillStatusType::kFormSignatureName, form_signature},
{UkmFieldFillStatusType::kFieldSignatureName, field_signature},
{UkmFieldFillStatusType::kValidationEventName, metric_type},
{UkmFieldTypeValidationType::kPredictionSourceName, source},
{UkmFieldTypeValidationType::kPredictedTypeName, predicted_type},
{UkmFieldTypeValidationType::kActualTypeName, actual_type}});
}
}
}
class MockAutofillClient : public TestAutofillClient {
public:
MockAutofillClient() {}
MOCK_METHOD1(ExecuteCommand, void(int));
};
} // namespace
// This is defined in the autofill_metrics.cc implementation file.
int GetFieldTypeGroupMetric(ServerFieldType field_type,
AutofillMetrics::FieldTypeQualityMetric metric);
class AutofillMetricsTest : public testing::Test {
public:
AutofillMetricsTest();
~AutofillMetricsTest() override;
void SetUp() override;
void TearDown() override;
protected:
void CreateAmbiguousProfiles();
// Removes all existing profiles and creates one profile.
void RecreateProfile();
// Removes all existing credit cards and creates a local, masked server,
// and/or full server credit card, according to the parameters.
void RecreateCreditCards(bool include_local_credit_card,
bool include_masked_server_credit_card,
bool include_full_server_credit_card);
// Removes all existing credit cards and creates 1 masked server card with a
// bank name.
void RecreateMaskedServerCreditCardWithBankName();
// Removes all existing credit cards and creates 1 full server card with a
// bank name.
void RecreateFullServerCreditCardWithBankName();
// Purge recorded UKM metrics for running more tests.
void PurgeUKM();
base::test::ScopedTaskEnvironment scoped_task_environment_;
ukm::TestAutoSetUkmRecorder test_ukm_recorder_;
MockAutofillClient autofill_client_;
syncer::TestSyncService sync_service_;
std::unique_ptr<TestAutofillDriver> autofill_driver_;
std::unique_ptr<TestAutofillManager> autofill_manager_;
std::unique_ptr<TestPersonalDataManager> personal_data_;
std::unique_ptr<MockAutocompleteHistoryManager> autocomplete_history_manager_;
std::unique_ptr<AutofillExternalDelegate> external_delegate_;
base::test::ScopedFeatureList scoped_feature_list_;
private:
void CreateTestAutofillProfiles();
};
AutofillMetricsTest::AutofillMetricsTest() {
autofill_driver_ = std::make_unique<TestAutofillDriver>();
}
AutofillMetricsTest::~AutofillMetricsTest() {
// Order of destruction is important as AutofillManager relies on
// PersonalDataManager to be around when it gets destroyed.
autofill_manager_.reset();
}
void AutofillMetricsTest::SetUp() {
autofill_client_.SetPrefs(test::PrefServiceForTesting());
personal_data_ = std::make_unique<TestPersonalDataManager>();
personal_data_->SetPrefService(autofill_client_.GetPrefs());
personal_data_->SetSyncServiceForTest(&sync_service_);
autocomplete_history_manager_ =
std::make_unique<MockAutocompleteHistoryManager>();
payments::TestPaymentsClient* payments_client =
new payments::TestPaymentsClient(
autofill_driver_->GetURLLoaderFactory(), autofill_client_.GetPrefs(),
autofill_client_.GetIdentityManager(), personal_data_.get());
autofill_client_.set_test_payments_client(
std::unique_ptr<payments::TestPaymentsClient>(payments_client));
TestCreditCardSaveManager* credit_card_save_manager =
new TestCreditCardSaveManager(autofill_driver_.get(), &autofill_client_,
payments_client, personal_data_.get());
autofill::TestFormDataImporter* test_form_data_importer =
new TestFormDataImporter(
&autofill_client_, payments_client,
std::unique_ptr<CreditCardSaveManager>(credit_card_save_manager),
personal_data_.get(), "en-US");
autofill_client_.set_test_form_data_importer(
std::unique_ptr<TestFormDataImporter>(test_form_data_importer));
autofill_manager_ = std::make_unique<TestAutofillManager>(
autofill_driver_.get(), &autofill_client_, personal_data_.get(),
autocomplete_history_manager_.get());
external_delegate_ = std::make_unique<AutofillExternalDelegate>(
autofill_manager_.get(), autofill_driver_.get());
autofill_manager_->SetExternalDelegate(external_delegate_.get());
// Initialize the TestPersonalDataManager with some default data.
CreateTestAutofillProfiles();
}
void AutofillMetricsTest::TearDown() {
// Order of destruction is important as AutofillManager relies on
// PersonalDataManager to be around when it gets destroyed.
autofill_manager_.reset();
autofill_driver_.reset();
personal_data_.reset();
test::ReenableSystemServices();
test_ukm_recorder_.Purge();
}
void AutofillMetricsTest::PurgeUKM() {
autofill_manager_->Reset();
test_ukm_recorder_.Purge();
autofill_client_.InitializeUKMSources();
}
void AutofillMetricsTest::CreateAmbiguousProfiles() {
personal_data_->ClearProfiles();
CreateTestAutofillProfiles();
AutofillProfile profile;
test::SetProfileInfo(&profile, "John", "Decca", "Public", "john@gmail.com",
"Company", "123 Main St.", "unit 7", "Springfield",
"Texas", "79401", "US", "2345678901");
profile.set_guid("00000000-0000-0000-0000-000000000003");
personal_data_->AddProfile(profile);
personal_data_->Refresh();
}
void AutofillMetricsTest::RecreateProfile() {
personal_data_->ClearProfiles();
AutofillProfile profile;
test::SetProfileInfo(&profile, "Elvis", "Aaron", "Presley",
"theking@gmail.com", "RCA", "3734 Elvis Presley Blvd.",
"Apt. 10", "Memphis", "Tennessee", "38116", "US",
"12345678901");
profile.set_guid("00000000-0000-0000-0000-000000000001");
personal_data_->AddProfile(profile);
personal_data_->Refresh();
}
void AutofillMetricsTest::RecreateCreditCards(
bool include_local_credit_card,
bool include_masked_server_credit_card,
bool include_full_server_credit_card) {
personal_data_->ClearCreditCards();
if (include_local_credit_card) {
CreditCard local_credit_card;
test::SetCreditCardInfo(&local_credit_card, "Test User",
"4111111111111111" /* Visa */, "11", "2022", "1");
local_credit_card.set_guid("10000000-0000-0000-0000-000000000001");
personal_data_->AddCreditCard(local_credit_card);
}
if (include_masked_server_credit_card) {
CreditCard masked_server_credit_card(CreditCard::MASKED_SERVER_CARD,
"server_id");
masked_server_credit_card.set_guid("10000000-0000-0000-0000-000000000002");
masked_server_credit_card.SetNetworkForMaskedCard(kDiscoverCard);
masked_server_credit_card.SetNumber(ASCIIToUTF16("9424"));
personal_data_->AddServerCreditCard(masked_server_credit_card);
}
if (include_full_server_credit_card) {
CreditCard full_server_credit_card(CreditCard::FULL_SERVER_CARD,
"server_id");
full_server_credit_card.set_guid("10000000-0000-0000-0000-000000000003");
personal_data_->AddFullServerCreditCard(full_server_credit_card);
}
personal_data_->Refresh();
}
void AutofillMetricsTest::RecreateMaskedServerCreditCardWithBankName() {
personal_data_->ClearCreditCards();
CreditCard credit_card(CreditCard::MASKED_SERVER_CARD, "server_id");
test::SetCreditCardInfo(&credit_card, "name", "1111" /* Visa */, "01", "2999",
"");
credit_card.set_guid("10000000-0000-0000-0000-000000000002");
credit_card.SetNetworkForMaskedCard(kVisaCard);
credit_card.set_bank_name("Chase");
personal_data_->AddServerCreditCard(credit_card);
personal_data_->Refresh();
}
void AutofillMetricsTest::RecreateFullServerCreditCardWithBankName() {
personal_data_->ClearCreditCards();
CreditCard credit_card(CreditCard::FULL_SERVER_CARD, "server_id");
test::SetCreditCardInfo(&credit_card, "name", "4111111111111111", "12", "24",
"1");
credit_card.set_guid("10000000-0000-0000-0000-000000000003");
credit_card.set_bank_name("Chase");
personal_data_->AddFullServerCreditCard(credit_card);
personal_data_->Refresh();
}
void AutofillMetricsTest::CreateTestAutofillProfiles() {
AutofillProfile profile1;
test::SetProfileInfo(&profile1, "Elvis", "Aaron", "Presley",
"theking@gmail.com", "RCA", "3734 Elvis Presley Blvd.",
"Apt. 10", "Memphis", "Tennessee", "38116", "US",
"12345678901");
profile1.set_guid("00000000-0000-0000-0000-000000000001");
personal_data_->AddProfile(profile1);
AutofillProfile profile2;
test::SetProfileInfo(&profile2, "Charles", "Hardin", "Holley",
"buddy@gmail.com", "Decca", "123 Apple St.", "unit 6",
"Lubbock", "Texas", "79401", "US", "2345678901");
profile2.set_guid("00000000-0000-0000-0000-000000000002");
personal_data_->AddProfile(profile2);
}
// Test parameter indicates if the metrics are being logged for a form in an
// iframe or the main frame. True means the form is in the main frame.
class AutofillMetricsIFrameTest : public AutofillMetricsTest,
public testing::WithParamInterface<bool> {
public:
AutofillMetricsIFrameTest()
: is_in_main_frame_(GetParam()),
credit_card_form_events_frame_histogram_(
std::string("Autofill.FormEvents.CreditCard.") +
(is_in_main_frame_ ? "IsInMainFrame" : "IsInIFrame")) {
autofill_driver_->SetIsInMainFrame(is_in_main_frame_);
}
protected:
const bool is_in_main_frame_;
const std::string credit_card_form_events_frame_histogram_;
};
INSTANTIATE_TEST_CASE_P(AutofillMetricsTest,
AutofillMetricsIFrameTest,
testing::Bool());
// Test parameter indicates if the metrics are being logged for a form in an
// iframe or the main frame. True means the form is in the main frame.
class AutofillMetricsCompanyTest : public AutofillMetricsTest,
public testing::WithParamInterface<bool> {
public:
AutofillMetricsCompanyTest() : company_name_enabled_(GetParam()) {}
void SetUp() override {
scoped_feature_list_.InitWithFeatureState(
features::kAutofillEnableCompanyName, company_name_enabled_);
AutofillMetricsTest::SetUp();
}
protected:
const bool company_name_enabled_;
private:
base::test::ScopedFeatureList scoped_feature_list_;
};
INSTANTIATE_TEST_CASE_P(AutofillMetricsTest,
AutofillMetricsCompanyTest,
testing::Bool());
// Test that we log quality metrics appropriately.
TEST_F(AutofillMetricsTest, QualityMetrics) {
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin = url::Origin::Create(autofill_client_.form_origin());
std::vector<ServerFieldType> heuristic_types, server_types;
FormFieldData field;
test::CreateTestFormField("Autofilled", "autofilled", "Elvis Aaron Presley",
"text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FIRST);
test::CreateTestFormField("Autofill Failed", "autofillfailed",
"buddy@gmail.com", "text", &field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_NUMBER);
server_types.push_back(EMAIL_ADDRESS);
test::CreateTestFormField("Empty", "empty", "", "text", &field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FIRST);
test::CreateTestFormField("Unknown", "unknown", "garbage", "text", &field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_NUMBER);
server_types.push_back(EMAIL_ADDRESS);
test::CreateTestFormField("Select", "select", "USA", "select-one", &field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(UNKNOWN_TYPE);
server_types.push_back(NO_SERVER_DATA);
test::CreateTestFormField("Phone", "phone", "2345678901", "tel", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
// Simulate having seen this form on page load.
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
// Simulate form submission.
base::HistogramTester histogram_tester;
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
// Heuristic predictions.
{
std::string aggregate_histogram =
"Autofill.FieldPredictionQuality.Aggregate.Heuristic";
std::string by_field_type_histogram =
"Autofill.FieldPredictionQuality.ByFieldType.Heuristic";
// Unknown:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_UNKNOWN, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(ADDRESS_HOME_COUNTRY,
AutofillMetrics::FALSE_NEGATIVE_UNKNOWN),
1);
// Match:
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::TRUE_POSITIVE, 2);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FULL, AutofillMetrics::TRUE_POSITIVE), 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_CITY_AND_NUMBER,
AutofillMetrics::TRUE_POSITIVE),
1);
// Mismatch:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_MISMATCH, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(EMAIL_ADDRESS,
AutofillMetrics::FALSE_NEGATIVE_MISMATCH),
1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_NUMBER,
AutofillMetrics::FALSE_POSITIVE_MISMATCH),
1);
// False Positive Unknown:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_POSITIVE_UNKNOWN, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_NUMBER,
AutofillMetrics::FALSE_POSITIVE_UNKNOWN),
1);
// False Positive Empty:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_POSITIVE_EMPTY, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FULL,
AutofillMetrics::FALSE_POSITIVE_EMPTY),
1);
// Sanity Check:
histogram_tester.ExpectTotalCount(aggregate_histogram, 6);
histogram_tester.ExpectTotalCount(by_field_type_histogram, 7);
}
// Server overrides heuristic so Overall and Server are the same predictions
// (as there were no test fields where server == NO_SERVER_DATA and heuristic
// != UNKNOWN_TYPE).
for (const std::string source : {"Server", "Overall"}) {
std::string aggregate_histogram =
"Autofill.FieldPredictionQuality.Aggregate." + source;
std::string by_field_type_histogram =
"Autofill.FieldPredictionQuality.ByFieldType." + source;
// Unknown:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_UNKNOWN, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(ADDRESS_HOME_COUNTRY,
AutofillMetrics::FALSE_NEGATIVE_UNKNOWN),
1);
// Match:
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::TRUE_POSITIVE, 2);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(EMAIL_ADDRESS, AutofillMetrics::TRUE_POSITIVE),
1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_WHOLE_NUMBER,
AutofillMetrics::TRUE_POSITIVE),
1);
// Mismatch:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_MISMATCH, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FULL,
AutofillMetrics::FALSE_NEGATIVE_MISMATCH),
1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FIRST,
AutofillMetrics::FALSE_POSITIVE_MISMATCH),
1);
// False Positive Unknown:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_POSITIVE_UNKNOWN, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(EMAIL_ADDRESS,
AutofillMetrics::FALSE_POSITIVE_UNKNOWN),
1);
// False Positive Empty:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_POSITIVE_EMPTY, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FIRST,
AutofillMetrics::FALSE_POSITIVE_EMPTY),
1);
// Sanity Check:
histogram_tester.ExpectTotalCount(aggregate_histogram, 6);
histogram_tester.ExpectTotalCount(by_field_type_histogram, 7);
}
}
// Test that we log quality metrics appropriately with fields having
// only_fill_when_focused and are supposed to log RATIONALIZATION_OK.
TEST_F(AutofillMetricsTest,
QualityMetrics_LoggedCorrecltyForRationalizationOk) {
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
std::vector<ServerFieldType> heuristic_types, server_types;
FormFieldData field;
test::CreateTestFormField("Name", "name", "Elvis Aaron Presley", "text",
&field);
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FULL);
test::CreateTestFormField("Address", "address", "3734 Elvis Presley Blvd.",
"text", &field);
form.fields.push_back(field);
heuristic_types.push_back(ADDRESS_HOME_LINE1);
server_types.push_back(ADDRESS_HOME_LINE1);
test::CreateTestFormField("Phone", "phone", "2345678901", "text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
field.is_autofilled = false;
// Below are fields with only_fill_when_focused set to true.
// RATIONALIZATION_OK because it's ambiguous value.
test::CreateTestFormField("Phone1", "phone1", "nonsense value", "text",
&field);
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_WHOLE_NUMBER);
// RATIONALIZATION_OK because it's same type but different
// to what is in the profile.
test::CreateTestFormField("Phone2", "phone2", "2345678902", "text", &field);
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
// RATIONALIZATION_OK because it's a type mismatch.
test::CreateTestFormField("Phone3", "phone3", "Elvis Aaron Presley", "text",
&field);
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_WHOLE_NUMBER);
base::UserActionTester user_action_tester;
// Simulate having seen this form on page load.
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
std::string guid("00000000-0000-0000-0000-000000000001");
// Trigger phone number rationalization at filling time.
autofill_manager_->FillOrPreviewForm(
AutofillDriver::FORM_DATA_ACTION_FILL, 0, form, form.fields.front(),
autofill_manager_->MakeFrontendID(std::string(), guid));
EXPECT_EQ(
1, user_action_tester.GetActionCount("Autofill_FilledProfileSuggestion"));
// Simulate form submission.
base::HistogramTester histogram_tester;
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
// Rationalization quality.
{
std::string rationalization_histogram =
"Autofill.RationalizationQuality.PhoneNumber";
// RATIONALIZATION_OK is logged 3 times.
histogram_tester.ExpectBucketCount(rationalization_histogram,
AutofillMetrics::RATIONALIZATION_OK, 3);
}
}
// Test that we log quality metrics appropriately with fields having
// only_fill_when_focused and are supposed to log RATIONALIZATION_GOOD.
TEST_F(AutofillMetricsTest,
QualityMetrics_LoggedCorrecltyForRationalizationGood) {
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
std::vector<ServerFieldType> heuristic_types, server_types;
FormFieldData field;
test::CreateTestFormField("Name", "name", "Elvis Aaron Presley", "text",
&field);
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FULL);
test::CreateTestFormField("Address", "address", "3734 Elvis Presley Blvd.",
"text", &field);
form.fields.push_back(field);
heuristic_types.push_back(ADDRESS_HOME_LINE1);
server_types.push_back(ADDRESS_HOME_LINE1);
test::CreateTestFormField("Phone", "phone", "2345678901", "text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
field.is_autofilled = false;
// Below are fields with only_fill_when_focused set to true.
// RATIONALIZATION_GOOD because it's empty.
test::CreateTestFormField("Phone1", "phone1", "", "text", &field);
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_WHOLE_NUMBER);
base::UserActionTester user_action_tester;
// Simulate having seen this form on page load.
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
std::string guid("00000000-0000-0000-0000-000000000001");
// Trigger phone number rationalization at filling time.
autofill_manager_->FillOrPreviewForm(
AutofillDriver::FORM_DATA_ACTION_FILL, 0, form, form.fields.front(),
autofill_manager_->MakeFrontendID(std::string(), guid));
EXPECT_EQ(
1, user_action_tester.GetActionCount("Autofill_FilledProfileSuggestion"));
// Simulate form submission.
base::HistogramTester histogram_tester;
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
// Rationalization quality.
{
std::string rationalization_histogram =
"Autofill.RationalizationQuality.PhoneNumber";
// RATIONALIZATION_GOOD is logged once.
histogram_tester.ExpectBucketCount(
rationalization_histogram, AutofillMetrics::RATIONALIZATION_GOOD, 1);
}
}
// Test that we log the skip decisions for hidden/representational fields
// correctly.
TEST_F(AutofillMetricsTest, LogHiddenRepresentationalFieldSkipDecision) {
// Create a profile.
RecreateProfile();
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin = url::Origin::Create(autofill_client_.form_origin());
FormFieldData field;
std::vector<ServerFieldType> field_types;
int64_t field_signature[4];
// no decision
test::CreateTestFormField("Name", "name", "", "text", &field);
form.fields.push_back(field);
field_types.push_back(NAME_FULL);
// skips
test::CreateTestFormField("Street", "street", "", "text", &field);
field.is_focusable = false;
form.fields.push_back(field);
field_types.push_back(ADDRESS_HOME_LINE1);
field_signature[0] = Collapse(CalculateFieldSignatureForField(field));
// skips
test::CreateTestFormField("City", "city", "", "text", &field);
field.role = FormFieldData::ROLE_ATTRIBUTE_PRESENTATION;
form.fields.push_back(field);
field_types.push_back(ADDRESS_HOME_CITY);
field_signature[1] = Collapse(CalculateFieldSignatureForField(field));
// doesn't skip
test::CreateTestFormField("State", "state", "", "select-one", &field);
field.is_focusable = false;
form.fields.push_back(field);
field_types.push_back(ADDRESS_HOME_STATE);
field_signature[2] = Collapse(CalculateFieldSignatureForField(field));
// doesn't skip
test::CreateTestFormField("Country", "country", "", "select-one", &field);
field.role = FormFieldData::ROLE_ATTRIBUTE_PRESENTATION;
form.fields.push_back(field);
field_types.push_back(ADDRESS_HOME_COUNTRY);
field_signature[3] = Collapse(CalculateFieldSignatureForField(field));
int64_t form_signature = Collapse(CalculateFormSignature(form));
// Simulate having seen this form on page load.
// |form_structure| will be owned by |autofill_manager_|.
autofill_manager_->AddSeenForm(form, field_types, field_types);
// Simulate filling form.
{
base::UserActionTester user_action_tester;
std::string guid("00000000-0000-0000-0000-000000000001"); // local profile.
autofill_manager_->FillOrPreviewForm(
AutofillDriver::FORM_DATA_ACTION_FILL, 0, form, form.fields.front(),
autofill_manager_->MakeFrontendID(std::string(), guid));
}
VerifyFormInteractionUkm(
test_ukm_recorder_, form,
UkmLogHiddenRepresentationalFieldSkipDecisionType::kEntryName,
{{{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFormSignatureName,
form_signature},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFieldSignatureName,
field_signature[0]},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFieldTypeGroupName,
ADDRESS_HOME},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::
kFieldOverallTypeName,
ADDRESS_HOME_LINE1},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHeuristicTypeName,
ADDRESS_HOME_LINE1},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kServerTypeName,
ADDRESS_HOME_LINE1},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHtmlFieldTypeName,
HTML_TYPE_UNSPECIFIED},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHtmlFieldModeName,
HTML_MODE_NONE},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kIsSkippedName,
true}},
{{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFormSignatureName,
form_signature},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFieldSignatureName,
field_signature[1]},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFieldTypeGroupName,
ADDRESS_HOME},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::
kFieldOverallTypeName,
ADDRESS_HOME_CITY},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHeuristicTypeName,
ADDRESS_HOME_CITY},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kServerTypeName,
ADDRESS_HOME_CITY},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHtmlFieldTypeName,
HTML_TYPE_UNSPECIFIED},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHtmlFieldModeName,
HTML_MODE_NONE},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kIsSkippedName,
true}},
{{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFormSignatureName,
form_signature},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFieldSignatureName,
field_signature[2]},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFieldTypeGroupName,
ADDRESS_HOME},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::
kFieldOverallTypeName,
ADDRESS_HOME_STATE},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHeuristicTypeName,
ADDRESS_HOME_STATE},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kServerTypeName,
ADDRESS_HOME_STATE},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHtmlFieldTypeName,
HTML_TYPE_UNSPECIFIED},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHtmlFieldModeName,
HTML_MODE_NONE},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kIsSkippedName,
false}},
{{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFormSignatureName,
form_signature},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFieldSignatureName,
field_signature[3]},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kFieldTypeGroupName,
ADDRESS_HOME},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::
kFieldOverallTypeName,
ADDRESS_HOME_COUNTRY},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHeuristicTypeName,
ADDRESS_HOME_COUNTRY},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kServerTypeName,
ADDRESS_HOME_COUNTRY},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHtmlFieldTypeName,
HTML_TYPE_UNSPECIFIED},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kHtmlFieldModeName,
HTML_MODE_NONE},
{UkmLogHiddenRepresentationalFieldSkipDecisionType::kIsSkippedName,
false}}});
}
// Test that we log the address line fields whose server types are rationalized
TEST_F(AutofillMetricsTest, LogRepeatedAddressTypeRationalized) {
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin = url::Origin::Create(autofill_client_.form_origin());
int64_t field_signature[2];
FormFieldData field;
field.form_control_type = "text";
field.label = ASCIIToUTF16("fullname");
field.name = ASCIIToUTF16("fullname");
form.fields.push_back(field);
field.label = ASCIIToUTF16("Street 1");
field.name = ASCIIToUTF16("street1");
form.fields.push_back(field);
field_signature[0] = Collapse(CalculateFieldSignatureForField(field));
field.label = ASCIIToUTF16("Street 2");
field.name = ASCIIToUTF16("street2");
form.fields.push_back(field);
field_signature[1] = Collapse(CalculateFieldSignatureForField(field));
int64_t form_signature = Collapse(CalculateFormSignature(form));
FormStructure form_structure(form);
std::vector<FormStructure*> forms;
forms.push_back(&form_structure);
std::vector<ServerFieldType> field_types;
for (size_t i = 0; i < forms[0]->field_count(); ++i)
field_types.push_back(UNKNOWN_TYPE);
// Simulate having seen this form on page load.
// |form_structure| will be owned by |autofill_manager_|.
autofill_manager_->AddSeenForm(form, field_types, field_types);
AutofillQueryResponseContents response;
response.add_field()->set_overall_type_prediction(NAME_FULL);
response.add_field()->set_overall_type_prediction(
ADDRESS_HOME_STREET_ADDRESS);
response.add_field()->set_overall_type_prediction(
ADDRESS_HOME_STREET_ADDRESS);
std::string response_string;
ASSERT_TRUE(response.SerializeToString(&response_string));
FormStructure::ParseQueryResponse(
response_string, forms,
autofill_manager_->form_interactions_ukm_logger());
ASSERT_EQ(test_ukm_recorder_
.GetEntriesByName(
UkmLogRepeatedServerTypePredictionRationalized::kEntryName)
.size(),
(size_t)2);
VerifyFormInteractionUkm(
test_ukm_recorder_, form,
UkmLogRepeatedServerTypePredictionRationalized::kEntryName,
{{{UkmLogRepeatedServerTypePredictionRationalized::kFormSignatureName,
form_signature},
{UkmLogRepeatedServerTypePredictionRationalized::kFieldSignatureName,
field_signature[0]},
{UkmLogRepeatedServerTypePredictionRationalized::kFieldTypeGroupName,
ADDRESS_HOME},
{UkmLogRepeatedServerTypePredictionRationalized::
kFieldOldOverallTypeName,
ADDRESS_HOME_STREET_ADDRESS},
{UkmLogRepeatedServerTypePredictionRationalized::kHeuristicTypeName,
UNKNOWN_TYPE},
{UkmLogRepeatedServerTypePredictionRationalized::kHtmlFieldTypeName,
HTML_TYPE_UNSPECIFIED},
{UkmLogRepeatedServerTypePredictionRationalized::kHtmlFieldModeName,
HTML_MODE_NONE},
{UkmLogRepeatedServerTypePredictionRationalized::
kFieldNewOverallTypeName,
ADDRESS_HOME_LINE1},
{UkmLogRepeatedServerTypePredictionRationalized::kServerTypeName,
ADDRESS_HOME_STREET_ADDRESS}},
{{UkmLogRepeatedServerTypePredictionRationalized::kFormSignatureName,
form_signature},
{UkmLogRepeatedServerTypePredictionRationalized::kFieldSignatureName,
field_signature[1]},
{UkmLogRepeatedServerTypePredictionRationalized::kFieldTypeGroupName,
ADDRESS_HOME},
{UkmLogRepeatedServerTypePredictionRationalized::
kFieldOldOverallTypeName,
ADDRESS_HOME_STREET_ADDRESS},
{UkmLogRepeatedServerTypePredictionRationalized::kHeuristicTypeName,
UNKNOWN_TYPE},
{UkmLogRepeatedServerTypePredictionRationalized::kHtmlFieldTypeName,
HTML_TYPE_UNSPECIFIED},
{UkmLogRepeatedServerTypePredictionRationalized::kHtmlFieldModeName,
HTML_MODE_NONE},
{UkmLogRepeatedServerTypePredictionRationalized::
kFieldNewOverallTypeName,
ADDRESS_HOME_LINE2},
{UkmLogRepeatedServerTypePredictionRationalized::kServerTypeName,
ADDRESS_HOME_STREET_ADDRESS}}});
}
// Test that we log the state/country fields whose server types are rationalized
TEST_F(AutofillMetricsTest, LogRepeatedStateCountryTypeRationalized) {
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin = url::Origin::Create(autofill_client_.form_origin());
int64_t field_signature[3];
FormFieldData field;
field.form_control_type = "text";
field.label = ASCIIToUTF16("Country");
field.name = ASCIIToUTF16("country");
form.fields.push_back(field);
field_signature[0] = Collapse(CalculateFieldSignatureForField(field));
field.label = ASCIIToUTF16("fullname");
field.name = ASCIIToUTF16("fullname");
form.fields.push_back(field);
field.label = ASCIIToUTF16("State");
field.name = ASCIIToUTF16("state");
form.fields.push_back(field);
field_signature[2] = Collapse(CalculateFieldSignatureForField(field));
field.label = ASCIIToUTF16("State");
field.name = ASCIIToUTF16("state");
field.is_focusable = false;
field.form_control_type = "select-one";
form.fields.push_back(field);
// Regardless of the order of appearance, hidden fields are rationalized
// before their corresponding visible one.
field_signature[1] = Collapse(CalculateFieldSignatureForField(field));
int64_t form_signature = Collapse(CalculateFormSignature(form));
FormStructure form_structure(form);
std::vector<FormStructure*> forms;
forms.push_back(&form_structure);
std::vector<ServerFieldType> field_types;
for (size_t i = 0; i < forms[0]->field_count(); ++i)
field_types.push_back(UNKNOWN_TYPE);
// Simulate having seen this form on page load.
// |form_structure| will be owned by |autofill_manager_|.
autofill_manager_->AddSeenForm(form, field_types, field_types);
AutofillQueryResponseContents response;
response.add_field()->set_overall_type_prediction(ADDRESS_HOME_COUNTRY);
response.add_field()->set_overall_type_prediction(NAME_FULL);
response.add_field()->set_overall_type_prediction(ADDRESS_HOME_COUNTRY);
response.add_field()->set_overall_type_prediction(ADDRESS_HOME_COUNTRY);
std::string response_string;
ASSERT_TRUE(response.SerializeToString(&response_string));
FormStructure::ParseQueryResponse(
response_string, forms,
autofill_manager_->form_interactions_ukm_logger());
ASSERT_EQ(test_ukm_recorder_
.GetEntriesByName(
UkmLogRepeatedServerTypePredictionRationalized::kEntryName)
.size(),
(size_t)3);
VerifyFormInteractionUkm(
test_ukm_recorder_, form,
UkmLogRepeatedServerTypePredictionRationalized::kEntryName,
{{{UkmLogRepeatedServerTypePredictionRationalized::kFormSignatureName,
form_signature},
{UkmLogRepeatedServerTypePredictionRationalized::kFieldSignatureName,
field_signature[0]},
{UkmLogRepeatedServerTypePredictionRationalized::kFieldTypeGroupName,
ADDRESS_HOME},
{UkmLogRepeatedServerTypePredictionRationalized::
kFieldOldOverallTypeName,
ADDRESS_HOME_COUNTRY},
{UkmLogRepeatedServerTypePredictionRationalized::kHeuristicTypeName,
UNKNOWN_TYPE},
{UkmLogRepeatedServerTypePredictionRationalized::kHtmlFieldTypeName,
HTML_TYPE_UNSPECIFIED},
{UkmLogRepeatedServerTypePredictionRationalized::kHtmlFieldModeName,
HTML_MODE_NONE},
{UkmLogRepeatedServerTypePredictionRationalized::kServerTypeName,
ADDRESS_HOME_COUNTRY},
{UkmLogRepeatedServerTypePredictionRationalized::
kFieldNewOverallTypeName,
ADDRESS_HOME_COUNTRY}},
{{UkmLogRepeatedServerTypePredictionRationalized::kFormSignatureName,
form_signature},
{UkmLogRepeatedServerTypePredictionRationalized::kFieldSignatureName,
field_signature[1]},
{UkmLogRepeatedServerTypePredictionRationalized::kFieldTypeGroupName,
ADDRESS_HOME},
{UkmLogRepeatedServerTypePredictionRationalized::
kFieldOldOverallTypeName,
ADDRESS_HOME_COUNTRY},
{UkmLogRepeatedServerTypePredictionRationalized::kHeuristicTypeName,
UNKNOWN_TYPE},
{UkmLogRepeatedServerTypePredictionRationalized::kHtmlFieldTypeName,
HTML_TYPE_UNSPECIFIED},
{UkmLogRepeatedServerTypePredictionRationalized::kHtmlFieldModeName,
HTML_MODE_NONE},
{UkmLogRepeatedServerTypePredictionRationalized::
kFieldNewOverallTypeName,
ADDRESS_HOME_STATE},
{UkmLogRepeatedServerTypePredictionRationalized::kServerTypeName,
ADDRESS_HOME_COUNTRY}},
{{UkmLogRepeatedServerTypePredictionRationalized::kFormSignatureName,
form_signature},
{UkmLogRepeatedServerTypePredictionRationalized::kFieldSignatureName,
field_signature[2]},
{UkmLogRepeatedServerTypePredictionRationalized::kFieldTypeGroupName,
ADDRESS_HOME},
{UkmLogRepeatedServerTypePredictionRationalized::
kFieldOldOverallTypeName,
ADDRESS_HOME_COUNTRY},
{UkmLogRepeatedServerTypePredictionRationalized::kHeuristicTypeName,
UNKNOWN_TYPE},
{UkmLogRepeatedServerTypePredictionRationalized::kHtmlFieldTypeName,
HTML_TYPE_UNSPECIFIED},
{UkmLogRepeatedServerTypePredictionRationalized::kHtmlFieldModeName,
HTML_MODE_NONE},
{UkmLogRepeatedServerTypePredictionRationalized::
kFieldNewOverallTypeName,
ADDRESS_HOME_STATE},
{UkmLogRepeatedServerTypePredictionRationalized::kServerTypeName,
ADDRESS_HOME_COUNTRY}}});
}
// Test that we log quality metrics appropriately with fields having
// only_fill_when_focused and are supposed to log RATIONALIZATION_BAD.
TEST_F(AutofillMetricsTest,
QualityMetrics_LoggedCorrecltyForRationalizationBad) {
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
std::vector<ServerFieldType> heuristic_types, server_types;
FormFieldData field;
test::CreateTestFormField("Name", "name", "Elvis Aaron Presley", "text",
&field);
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FULL);
test::CreateTestFormField("Address", "address", "3734 Elvis Presley Blvd.",
"text", &field);
form.fields.push_back(field);
heuristic_types.push_back(ADDRESS_HOME_LINE1);
server_types.push_back(ADDRESS_HOME_LINE1);
test::CreateTestFormField("Phone", "phone", "2345678901", "text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
field.is_autofilled = false;
// Below are fields with only_fill_when_focused set to true.
// RATIONALIZATION_BAD because it's filled with same
// value as filled previously.
test::CreateTestFormField("Phone1", "phone1", "2345678901", "text", &field);
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_WHOLE_NUMBER);
base::UserActionTester user_action_tester;
// Simulate having seen this form on page load.
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
std::string guid("00000000-0000-0000-0000-000000000001");
// Trigger phone number rationalization at filling time.
autofill_manager_->FillOrPreviewForm(
AutofillDriver::FORM_DATA_ACTION_FILL, 0, form, form.fields.front(),
autofill_manager_->MakeFrontendID(std::string(), guid));
EXPECT_EQ(
1, user_action_tester.GetActionCount("Autofill_FilledProfileSuggestion"));
// Simulate form submission.
base::HistogramTester histogram_tester;
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
// Rationalization quality.
{
std::string rationalization_histogram =
"Autofill.RationalizationQuality.PhoneNumber";
// RATIONALIZATION_BAD is logged once.
histogram_tester.ExpectBucketCount(rationalization_histogram,
AutofillMetrics::RATIONALIZATION_BAD, 1);
}
}
// Test that we log quality metrics appropriately with fields having
// only_fill_when_focused set to true.
TEST_F(AutofillMetricsTest,
QualityMetrics_LoggedCorrecltyForOnlyFillWhenFocusedField) {
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
std::vector<ServerFieldType> heuristic_types, server_types;
FormFieldData field;
// TRUE_POSITIVE + no rationalization logging
test::CreateTestFormField("Name", "name", "Elvis Aaron Presley", "text",
&field);
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FULL);
// TRUE_POSITIVE + no rationalization logging
test::CreateTestFormField("Address", "address", "3734 Elvis Presley Blvd.",
"text", &field);
form.fields.push_back(field);
heuristic_types.push_back(ADDRESS_HOME_LINE1);
server_types.push_back(ADDRESS_HOME_LINE1);
// TRUE_POSITIVE + no rationalization logging
test::CreateTestFormField("Phone", "phone", "2345678901", "text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
field.is_autofilled = false;
// Below are fields with only_fill_when_focused set to true.
// TRUE_NEGATIVE_EMPTY + RATIONALIZATION_GOOD
test::CreateTestFormField("Phone1", "phone1", "", "text", &field);
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_WHOLE_NUMBER);
// TRUE_POSITIVE + RATIONALIZATION_BAD
test::CreateTestFormField("Phone2", "phone2", "2345678901", "text", &field);
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
// FALSE_NEGATIVE_MISMATCH + RATIONALIZATION_OK
test::CreateTestFormField("Phone3", "phone3", "Elvis Aaron Presley", "text",
&field);
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_WHOLE_NUMBER);
base::UserActionTester user_action_tester;
// Simulate having seen this form on page load.
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
std::string guid("00000000-0000-0000-0000-000000000001");
// Trigger phone number rationalization at filling time.
autofill_manager_->FillOrPreviewForm(
AutofillDriver::FORM_DATA_ACTION_FILL, 0, form, form.fields.front(),
autofill_manager_->MakeFrontendID(std::string(), guid));
EXPECT_EQ(
1, user_action_tester.GetActionCount("Autofill_FilledProfileSuggestion"));
// Simulate form submission.
base::HistogramTester histogram_tester;
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
// Rationalization quality.
{
std::string rationalization_histogram =
"Autofill.RationalizationQuality.PhoneNumber";
histogram_tester.ExpectBucketCount(
rationalization_histogram, AutofillMetrics::RATIONALIZATION_GOOD, 1);
histogram_tester.ExpectBucketCount(rationalization_histogram,
AutofillMetrics::RATIONALIZATION_OK, 1);
histogram_tester.ExpectBucketCount(rationalization_histogram,
AutofillMetrics::RATIONALIZATION_BAD, 1);
}
// Heuristic predictions.
{
std::string aggregate_histogram =
"Autofill.FieldPredictionQuality.Aggregate.Heuristic";
std::string by_field_type_histogram =
"Autofill.FieldPredictionQuality.ByFieldType.Heuristic";
// TRUE_POSITIVE:
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::TRUE_POSITIVE, 4);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FULL, AutofillMetrics::TRUE_POSITIVE), 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(ADDRESS_HOME_LINE1,
AutofillMetrics::TRUE_POSITIVE),
1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_CITY_AND_NUMBER,
AutofillMetrics::TRUE_POSITIVE),
2);
// TRUE_NEGATIVE_EMPTY
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::TRUE_NEGATIVE_EMPTY, 1);
// FALSE_NEGATIVE_MISMATCH
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_MISMATCH, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_WHOLE_NUMBER,
AutofillMetrics::FALSE_NEGATIVE_MISMATCH),
1);
// Sanity Check:
histogram_tester.ExpectTotalCount(aggregate_histogram, 6);
histogram_tester.ExpectTotalCount(by_field_type_histogram, 5);
}
// Server overrides heuristic so Overall and Server are the same predictions
// (as there were no test fields where server == NO_SERVER_DATA and heuristic
// != UNKNOWN_TYPE).
for (const std::string source : {"Server", "Overall"}) {
std::string aggregate_histogram =
"Autofill.FieldPredictionQuality.Aggregate." + source;
std::string by_field_type_histogram =
"Autofill.FieldPredictionQuality.ByFieldType." + source;
// TRUE_POSITIVE:
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::TRUE_POSITIVE, 4);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FULL, AutofillMetrics::TRUE_POSITIVE), 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(ADDRESS_HOME_LINE1,
AutofillMetrics::TRUE_POSITIVE),
1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_CITY_AND_NUMBER,
AutofillMetrics::TRUE_POSITIVE),
2);
// TRUE_NEGATIVE_EMPTY
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::TRUE_NEGATIVE_EMPTY, 1);
// FALSE_NEGATIVE_MISMATCHFALSE_NEGATIVE_MATCH
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_MISMATCH, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_CITY_AND_NUMBER,
AutofillMetrics::FALSE_NEGATIVE_MISMATCH),
1);
// Sanity Check:
histogram_tester.ExpectTotalCount(aggregate_histogram, 6);
histogram_tester.ExpectTotalCount(by_field_type_histogram, 5);
}
}
// Tests the true negatives (empty + no prediction and unknown + no prediction)
// and false positives (empty + bad prediction and unknown + bad prediction)
// are counted correctly.
struct QualityMetricsTestCase {
const ServerFieldType predicted_field_type;
const ServerFieldType actual_field_type;
};
class QualityMetricsTest
: public AutofillMetricsTest,
public testing::WithParamInterface<QualityMetricsTestCase> {
public:
const char* ValueForType(ServerFieldType type) {
switch (type) {
case EMPTY_TYPE:
return "";
case NO_SERVER_DATA:
case UNKNOWN_TYPE:
return "unknown";
case COMPANY_NAME:
return "RCA";
case NAME_FIRST:
return "Elvis";
case NAME_MIDDLE:
return "Aaron";
case NAME_LAST:
return "Presley";
case NAME_FULL:
return "Elvis Aaron Presley";
case EMAIL_ADDRESS:
return "buddy@gmail.com";
case PHONE_HOME_NUMBER:
case PHONE_HOME_WHOLE_NUMBER:
case PHONE_HOME_CITY_AND_NUMBER:
return "2345678901";
case ADDRESS_HOME_STREET_ADDRESS:
return "123 Apple St.\nunit 6";
case ADDRESS_HOME_LINE1:
return "123 Apple St.";
case ADDRESS_HOME_LINE2:
return "unit 6";
case ADDRESS_HOME_CITY:
return "Lubbock";
case ADDRESS_HOME_STATE:
return "Texas";
case ADDRESS_HOME_ZIP:
return "79401";
case ADDRESS_HOME_COUNTRY:
return "US";
case AMBIGUOUS_TYPE:
// This occurs as both a company name and a middle name once ambiguous
// profiles are created.
CreateAmbiguousProfiles();
return "Decca";
default:
NOTREACHED(); // Fall through
return "unexpected!";
}
}
bool IsExampleOf(AutofillMetrics::FieldTypeQualityMetric metric,
ServerFieldType predicted_type,
ServerFieldType actual_type) {
// The server can send either NO_SERVER_DATA or UNKNOWN_TYPE to indicate
// that a field is not autofillable:
//
// NO_SERVER_DATA
// => A type cannot be determined based on available data.
// UNKNOWN_TYPE
// => field is believed to not have an autofill type.
//
// Both of these are tabulated as "negative" predictions; so, to simplify
// the logic below, map them both to UNKNOWN_TYPE.
if (predicted_type == NO_SERVER_DATA)
predicted_type = UNKNOWN_TYPE;
switch (metric) {
case AutofillMetrics::TRUE_POSITIVE:
return unknown_equivalent_types_.count(actual_type) == 0 &&
predicted_type == actual_type;
case AutofillMetrics::TRUE_NEGATIVE_AMBIGUOUS:
return actual_type == AMBIGUOUS_TYPE && predicted_type == UNKNOWN_TYPE;
case AutofillMetrics::TRUE_NEGATIVE_UNKNOWN:
return actual_type == UNKNOWN_TYPE && predicted_type == UNKNOWN_TYPE;
case AutofillMetrics::TRUE_NEGATIVE_EMPTY:
return actual_type == EMPTY_TYPE && predicted_type == UNKNOWN_TYPE;
case AutofillMetrics::FALSE_POSITIVE_AMBIGUOUS:
return actual_type == AMBIGUOUS_TYPE && predicted_type != UNKNOWN_TYPE;
case AutofillMetrics::FALSE_POSITIVE_UNKNOWN:
return actual_type == UNKNOWN_TYPE && predicted_type != UNKNOWN_TYPE;
case AutofillMetrics::FALSE_POSITIVE_EMPTY:
return actual_type == EMPTY_TYPE && predicted_type != UNKNOWN_TYPE;
// False negative mismatch and false positive mismatch trigger on the same
// conditions:
// - False positive prediction of predicted type
// - False negative prediction of actual type
case AutofillMetrics::FALSE_POSITIVE_MISMATCH:
case AutofillMetrics::FALSE_NEGATIVE_MISMATCH:
return unknown_equivalent_types_.count(actual_type) == 0 &&
actual_type != predicted_type && predicted_type != UNKNOWN_TYPE;
case AutofillMetrics::FALSE_NEGATIVE_UNKNOWN:
return unknown_equivalent_types_.count(actual_type) == 0 &&
actual_type != predicted_type && predicted_type == UNKNOWN_TYPE;
default:
NOTREACHED();
}
return false;
}
static int FieldTypeCross(ServerFieldType predicted_type,
ServerFieldType actual_type) {
EXPECT_LE(predicted_type, UINT16_MAX);
EXPECT_LE(actual_type, UINT16_MAX);
return (predicted_type << 16) | actual_type;
}
const ServerFieldTypeSet unknown_equivalent_types_{UNKNOWN_TYPE, EMPTY_TYPE,
AMBIGUOUS_TYPE};
};
TEST_P(QualityMetricsTest, Classification) {
const std::vector<std::string> prediction_sources{"Heuristic", "Server",
"Overall"};
// Setup the test parameters.
ServerFieldType actual_field_type = GetParam().actual_field_type;
ServerFieldType predicted_type = GetParam().predicted_field_type;
DVLOG(2) << "Test Case = Predicted: "
<< AutofillType(predicted_type).ToString() << "; "
<< "Actual: " << AutofillType(actual_field_type).ToString();
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin = url::Origin::Create(autofill_client_.form_origin());
std::vector<ServerFieldType> heuristic_types, server_types, actual_types;
AutofillField field;
// Add a first name field, that is predicted correctly.
test::CreateTestFormField("first", "first", ValueForType(NAME_FIRST), "text",
&field);
form.fields.push_back(field);
heuristic_types.push_back(NAME_FIRST);
server_types.push_back(NAME_FIRST);
actual_types.push_back(NAME_FIRST);
// Add a last name field, that is predicted correctly.
test::CreateTestFormField("last", "last", ValueForType(NAME_LAST), "test",
&field);
form.fields.push_back(field);
heuristic_types.push_back(NAME_LAST);
server_types.push_back(NAME_LAST);
actual_types.push_back(NAME_LAST);
// Add an empty or unknown field, that is predicted as per the test params.
test::CreateTestFormField("Unknown", "Unknown",
ValueForType(actual_field_type), "text", &field);
form.fields.push_back(field);
heuristic_types.push_back(predicted_type == NO_SERVER_DATA ? UNKNOWN_TYPE
: predicted_type);
server_types.push_back(predicted_type);
// Resolve any field type ambiguity.
if (actual_field_type == AMBIGUOUS_TYPE) {
if (predicted_type == COMPANY_NAME || predicted_type == NAME_MIDDLE)
actual_field_type = predicted_type;
}
actual_types.push_back(actual_field_type);
// Simulate having seen this form on page load.
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
// Run the form submission code while tracking the histograms.
base::HistogramTester histogram_tester;
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
ExpectedUkmMetrics expected_ukm_metrics;
AppendFieldTypeUkm(form, heuristic_types, server_types, actual_types,
&expected_ukm_metrics);
VerifyFormInteractionUkm(test_ukm_recorder_, form,
UkmFieldTypeValidationType::kEntryName,
expected_ukm_metrics);
// Validate the total samples and the crossed (predicted-to-actual) samples.
for (const auto& source : prediction_sources) {
const std::string crossed_histogram = "Autofill.FieldPrediction." + source;
const std::string aggregate_histogram =
"Autofill.FieldPredictionQuality.Aggregate." + source;
const std::string by_field_type_histogram =
"Autofill.FieldPredictionQuality.ByFieldType." + source;
// Sanity Check:
histogram_tester.ExpectTotalCount(crossed_histogram, 3);
histogram_tester.ExpectTotalCount(aggregate_histogram, 3);
histogram_tester.ExpectTotalCount(
by_field_type_histogram,
2 +
(predicted_type != UNKNOWN_TYPE &&
predicted_type != NO_SERVER_DATA &&
predicted_type != actual_field_type) +
(unknown_equivalent_types_.count(actual_field_type) == 0));
// The Crossed Histogram:
histogram_tester.ExpectBucketCount(
crossed_histogram, FieldTypeCross(NAME_FIRST, NAME_FIRST), 1);
histogram_tester.ExpectBucketCount(crossed_histogram,
FieldTypeCross(NAME_LAST, NAME_LAST), 1);
histogram_tester.ExpectBucketCount(
crossed_histogram,
FieldTypeCross((predicted_type == NO_SERVER_DATA && source != "Server"
? UNKNOWN_TYPE
: predicted_type),
actual_field_type),
1);
}
// Validate the individual histogram counter values.
for (int i = 0; i < AutofillMetrics::NUM_FIELD_TYPE_QUALITY_METRICS; ++i) {
// The metric enum value we're currently examining.
auto metric = static_cast<AutofillMetrics::FieldTypeQualityMetric>(i);
// The type specific expected count is 1 if (predicted, actual) is an
// example
int basic_expected_count =
IsExampleOf(metric, predicted_type, actual_field_type) ? 1 : 0;
// For aggregate metrics don't capture aggregate FALSE_POSITIVE_MISMATCH.
// Note there are two true positive values (first and last name) hard-
// coded into the test.
int aggregate_expected_count =
(metric == AutofillMetrics::TRUE_POSITIVE ? 2 : 0) +
(metric == AutofillMetrics::FALSE_POSITIVE_MISMATCH
? 0
: basic_expected_count);
// If this test exercises the ambiguous middle name match, then validation
// of the name-specific metrics must include the true-positives created by
// the first and last name fields.
if (metric == AutofillMetrics::TRUE_POSITIVE &&
predicted_type == NAME_MIDDLE && actual_field_type == NAME_MIDDLE) {
basic_expected_count += 2;
}
// For metrics keyed to the actual field type, we don't capture unknown,
// empty or ambiguous and we don't capture false positive mismatches.
int expected_count_for_actual_type =
(unknown_equivalent_types_.count(actual_field_type) == 0 &&
metric != AutofillMetrics::FALSE_POSITIVE_MISMATCH)
? basic_expected_count
: 0;
// For metrics keyed to the predicted field type, we don't capture unknown
// (empty is not a predictable value) and we don't capture false negative
// mismatches.
int expected_count_for_predicted_type =
(predicted_type != UNKNOWN_TYPE && predicted_type != NO_SERVER_DATA &&
metric != AutofillMetrics::FALSE_NEGATIVE_MISMATCH)
? basic_expected_count
: 0;
for (const auto& source : prediction_sources) {
std::string aggregate_histogram =
"Autofill.FieldPredictionQuality.Aggregate." + source;
std::string by_field_type_histogram =
"Autofill.FieldPredictionQuality.ByFieldType." + source;
histogram_tester.ExpectBucketCount(aggregate_histogram, metric,
aggregate_expected_count);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(actual_field_type, metric),
expected_count_for_actual_type);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(predicted_type, metric),
expected_count_for_predicted_type);
}
}
}
INSTANTIATE_TEST_CASE_P(
AutofillMetricsTest,
QualityMetricsTest,
testing::Values(QualityMetricsTestCase{NO_SERVER_DATA, EMPTY_TYPE},
QualityMetricsTestCase{NO_SERVER_DATA, UNKNOWN_TYPE},
QualityMetricsTestCase{NO_SERVER_DATA, AMBIGUOUS_TYPE},
QualityMetricsTestCase{NO_SERVER_DATA, EMAIL_ADDRESS},
QualityMetricsTestCase{EMAIL_ADDRESS, EMPTY_TYPE},
QualityMetricsTestCase{EMAIL_ADDRESS, UNKNOWN_TYPE},
QualityMetricsTestCase{EMAIL_ADDRESS, AMBIGUOUS_TYPE},
QualityMetricsTestCase{EMAIL_ADDRESS, EMAIL_ADDRESS},
QualityMetricsTestCase{EMAIL_ADDRESS, COMPANY_NAME},
QualityMetricsTestCase{COMPANY_NAME, EMAIL_ADDRESS},
QualityMetricsTestCase{NAME_MIDDLE, AMBIGUOUS_TYPE},
QualityMetricsTestCase{COMPANY_NAME, AMBIGUOUS_TYPE},
QualityMetricsTestCase{UNKNOWN_TYPE, EMPTY_TYPE},
QualityMetricsTestCase{UNKNOWN_TYPE, UNKNOWN_TYPE},
QualityMetricsTestCase{UNKNOWN_TYPE, AMBIGUOUS_TYPE},
QualityMetricsTestCase{UNKNOWN_TYPE, EMAIL_ADDRESS}));
// Ensures that metrics that measure timing some important Autofill functions
// actually are recorded and retrieved.
TEST_F(AutofillMetricsTest, TimingMetrics) {
base::HistogramTester histogram_tester;
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin =
url::Origin::Create(GURL("http://example_root.com/form.html"));
FormFieldData field;
test::CreateTestFormField("Autofilled", "autofilled", "Elvis Aaron Presley",
"text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
test::CreateTestFormField("Autofill Failed", "autofillfailed",
"buddy@gmail.com", "text", &field);
field.is_autofilled = false;
form.fields.push_back(field);
test::CreateTestFormField("Phone", "phone", "2345678901", "tel", &field);
field.is_autofilled = false;
form.fields.push_back(field);
// Simulate a OnFormsSeen() call that should trigger the recording.
std::vector<FormData> forms;
forms.push_back(form);
autofill_manager_->OnFormsSeen(forms, TimeTicks::Now());
// Because these metrics are related to timing, it is not possible to know in
// advance which bucket the sample will fall into, so we just need to make
// sure we have valid samples.
EXPECT_FALSE(
histogram_tester.GetAllSamples("Autofill.Timing.DetermineHeuristicTypes")
.empty());
EXPECT_FALSE(
histogram_tester.GetAllSamples("Autofill.Timing.ParseForm").empty());
}
// Test that we log quality metrics appropriately when an upload is triggered
// but no submission event is sent.
TEST_F(AutofillMetricsTest, QualityMetrics_NoSubmission) {
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin = url::Origin::Create(autofill_client_.form_origin());
std::vector<ServerFieldType> heuristic_types, server_types;
FormFieldData field;
test::CreateTestFormField("Autofilled", "autofilled", "Elvis Aaron Presley",
"text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FIRST);
test::CreateTestFormField("Autofill Failed", "autofillfailed",
"buddy@gmail.com", "text", &field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_NUMBER);
server_types.push_back(EMAIL_ADDRESS);
test::CreateTestFormField("Empty", "empty", "", "text", &field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FIRST);
test::CreateTestFormField("Unknown", "unknown", "garbage", "text", &field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_NUMBER);
server_types.push_back(EMAIL_ADDRESS);
test::CreateTestFormField("Select", "select", "USA", "select-one", &field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(UNKNOWN_TYPE);
server_types.push_back(NO_SERVER_DATA);
test::CreateTestFormField("Phone", "phone", "2345678901", "tel", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
// Simulate having seen this form on page load.
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
// Simulate text input on one of the fields.
autofill_manager_->OnTextFieldDidChange(form, form.fields[0], gfx::RectF(),
TimeTicks());
// Trigger a form upload and metrics by Resetting the manager.
base::HistogramTester histogram_tester;
autofill_manager_->Reset();
// Heuristic predictions.
{
std::string aggregate_histogram =
"Autofill.FieldPredictionQuality.Aggregate.Heuristic.NoSubmission";
std::string by_field_type_histogram =
"Autofill.FieldPredictionQuality.ByFieldType.Heuristic.NoSubmission";
// False Negative:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_UNKNOWN, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(ADDRESS_HOME_COUNTRY,
AutofillMetrics::FALSE_NEGATIVE_UNKNOWN),
1);
// Match:
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::TRUE_POSITIVE, 2);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FULL, AutofillMetrics::TRUE_POSITIVE), 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_WHOLE_NUMBER,
AutofillMetrics::TRUE_POSITIVE),
1);
// Mismatch:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_MISMATCH, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(EMAIL_ADDRESS,
AutofillMetrics::FALSE_NEGATIVE_MISMATCH),
1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_NUMBER,
AutofillMetrics::FALSE_POSITIVE_MISMATCH),
1);
// False Positives:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_POSITIVE_EMPTY, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FULL,
AutofillMetrics::FALSE_POSITIVE_EMPTY),
1);
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_POSITIVE_UNKNOWN, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_NUMBER,
AutofillMetrics::FALSE_POSITIVE_UNKNOWN),
1);
// Sanity Check:
histogram_tester.ExpectTotalCount(aggregate_histogram, 6);
histogram_tester.ExpectTotalCount(by_field_type_histogram, 7);
}
// Server predictions override heuristics, so server and overall will be the
// same.
for (const std::string source : {"Server", "Overall"}) {
std::string aggregate_histogram =
"Autofill.FieldPredictionQuality.Aggregate." + source + ".NoSubmission";
std::string by_field_type_histogram =
"Autofill.FieldPredictionQuality.ByFieldType." + source +
".NoSubmission";
// Unknown.
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_UNKNOWN, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(ADDRESS_HOME_COUNTRY,
AutofillMetrics::FALSE_NEGATIVE_UNKNOWN),
1);
// Match:
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::TRUE_POSITIVE, 2);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(EMAIL_ADDRESS, AutofillMetrics::TRUE_POSITIVE),
1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(PHONE_HOME_WHOLE_NUMBER,
AutofillMetrics::TRUE_POSITIVE),
1);
// Mismatch:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_MISMATCH, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FULL,
AutofillMetrics::FALSE_NEGATIVE_MISMATCH),
1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FIRST,
AutofillMetrics::FALSE_POSITIVE_MISMATCH),
1);
// False Positives:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_POSITIVE_EMPTY, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FIRST,
AutofillMetrics::FALSE_POSITIVE_EMPTY),
1);
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_POSITIVE_UNKNOWN, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(EMAIL_ADDRESS,
AutofillMetrics::FALSE_POSITIVE_UNKNOWN),
1);
// Sanity Check:
histogram_tester.ExpectTotalCount(aggregate_histogram, 6);
histogram_tester.ExpectTotalCount(by_field_type_histogram, 7);
}
}
// Test that we log quality metrics for heuristics and server predictions based
// on autocomplete attributes present on the fields.
TEST_F(AutofillMetricsTest, QualityMetrics_BasedOnAutocomplete) {
FormData form;
form.name = ASCIIToUTF16("MyForm");
form.origin = GURL("http://myform.com/form.html");
form.action = GURL("http://myform.com/submit.html");
form.main_frame_origin =
url::Origin::Create(GURL("http://example_root.com/form.html"));
FormFieldData field;
// Heuristic value will match with Autocomplete attribute.
test::CreateTestFormField("Last Name", "lastname", "", "text", &field);
field.autocomplete_attribute = "family-name";
form.fields.push_back(field);
// Heuristic value will NOT match with Autocomplete attribute.
test::CreateTestFormField("First Name", "firstname", "", "text", &field);
field.autocomplete_attribute = "additional-name";
form.fields.push_back(field);
// Heuristic value will be unknown.
test::CreateTestFormField("Garbage label", "garbage", "", "text", &field);
field.autocomplete_attribute = "postal-code";
form.fields.push_back(field);
// No autocomplete attribute. No metric logged.
test::CreateTestFormField("Address", "address", "", "text", &field);
field.autocomplete_attribute = "";
form.fields.push_back(field);
std::unique_ptr<TestFormStructure> form_structure =
std::make_unique<TestFormStructure>(form);
TestFormStructure* form_structure_ptr = form_structure.get();
form_structure->DetermineHeuristicTypes();
ASSERT_TRUE(autofill_manager_->mutable_form_structures()
->emplace(form_structure_ptr->form_signature(),
std::move(form_structure))
.second);
AutofillQueryResponseContents response;
// Server response will match with autocomplete.
response.add_field()->set_overall_type_prediction(NAME_LAST);
// Server response will NOT match with autocomplete.
response.add_field()->set_overall_type_prediction(NAME_FIRST);
// Server response will have no data.
response.add_field()->set_overall_type_prediction(NO_SERVER_DATA);
// Not logged.
response.add_field()->set_overall_type_prediction(NAME_MIDDLE);
std::string response_string;
ASSERT_TRUE(response.SerializeToString(&response_string));
std::vector<std::string> signatures;
signatures.push_back(form_structure_ptr->FormSignatureAsStr());
base::HistogramTester histogram_tester;
autofill_manager_->OnLoadedServerPredictions(response_string, signatures);
// Verify that FormStructure::ParseQueryResponse was called (here and below).
histogram_tester.ExpectBucketCount("Autofill.ServerQueryResponse",
AutofillMetrics::QUERY_RESPONSE_RECEIVED,
1);
histogram_tester.ExpectBucketCount("Autofill.ServerQueryResponse",
AutofillMetrics::QUERY_RESPONSE_PARSED, 1);
// Autocomplete-derived types are eventually what's inferred.
EXPECT_EQ(NAME_LAST, form_structure_ptr->field(0)->Type().GetStorableType());
EXPECT_EQ(NAME_MIDDLE,
form_structure_ptr->field(1)->Type().GetStorableType());
EXPECT_EQ(ADDRESS_HOME_ZIP,
form_structure_ptr->field(2)->Type().GetStorableType());
for (const std::string source : {"Heuristic", "Server"}) {
std::string aggregate_histogram =
"Autofill.FieldPredictionQuality.Aggregate." + source +
".BasedOnAutocomplete";
std::string by_field_type_histogram =
"Autofill.FieldPredictionQuality.ByFieldType." + source +
".BasedOnAutocomplete";
// Unknown:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_UNKNOWN, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(ADDRESS_HOME_ZIP,
AutofillMetrics::FALSE_NEGATIVE_UNKNOWN),
1);
// Match:
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::TRUE_POSITIVE, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_LAST, AutofillMetrics::TRUE_POSITIVE), 1);
// Mismatch:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_MISMATCH, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FIRST,
AutofillMetrics::FALSE_POSITIVE_MISMATCH),
1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_MIDDLE,
AutofillMetrics::FALSE_POSITIVE_MISMATCH),
1);
// Sanity check.
histogram_tester.ExpectTotalCount(aggregate_histogram, 3);
histogram_tester.ExpectTotalCount(by_field_type_histogram, 4);
}
}
// Test that we log UPI Virtual Payment Address.
TEST_F(AutofillMetricsTest, UpiVirtualPaymentAddress) {
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin =
url::Origin::Create(GURL("http://example_root.com/form.html"));
std::vector<ServerFieldType> heuristic_types, server_types;
FormFieldData field;
// Heuristic value will match with Autocomplete attribute.
test::CreateTestFormField("Last Name", "lastname", "", "text", &field);
form.fields.push_back(field);
heuristic_types.push_back(NAME_LAST);
server_types.push_back(NAME_LAST);
// Heuristic value will NOT match with Autocomplete attribute.
test::CreateTestFormField("First Name", "firstname", "", "text", &field);
form.fields.push_back(field);
heuristic_types.push_back(NAME_FIRST);
server_types.push_back(NAME_FIRST);
// Heuristic value will NOT match with Autocomplete attribute.
test::CreateTestFormField("Payment Address", "payment_address", "user@upi",
"text", &field);
form.fields.push_back(field);
heuristic_types.push_back(ADDRESS_HOME_LINE1);
server_types.push_back(ADDRESS_HOME_LINE1);
// Simulate having seen this form on page load.
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
// Simulate form submission.
base::HistogramTester histogram_tester;
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
histogram_tester.ExpectBucketCount(
"Autofill.UserHappiness", AutofillMetrics::USER_DID_ENTER_UPI_VPA, 1);
histogram_tester.ExpectBucketCount("Autofill.UserHappiness.Address",
AutofillMetrics::USER_DID_ENTER_UPI_VPA,
1);
histogram_tester.ExpectTotalCount("Autofill.UserHappiness.CreditCard", 0);
histogram_tester.ExpectTotalCount("Autofill.UserHappiness.Password", 0);
histogram_tester.ExpectTotalCount("Autofill.UserHappiness.Unknown", 0);
}
// Verify that when a field is annotated with the autocomplete attribute, its
// predicted type is remembered when quality metrics are logged.
TEST_F(AutofillMetricsTest, PredictedMetricsWithAutocomplete) {
// Allow heuristics to run (and be accepted) for small forms.
base::test::ScopedFeatureList feature_list;
feature_list.InitAndDisableFeature(
kAutofillEnforceMinRequiredFieldsForHeuristics);
// Set up our form data. Note that the fields have default values not found
// in the user profiles. They will be changed between the time the form is
// seen/parsed, and the time it is submitted.
FormData form;
FormFieldData field;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin =
url::Origin::Create(GURL("http://example_root.com/form.html"));
test::CreateTestFormField("Select", "select", "USA", "select-one", &field);
form.fields.push_back(field);
form.fields.back().autocomplete_attribute = "country";
test::CreateTestFormField("Unknown", "Unknown", "", "text", &field);
form.fields.push_back(field);
test::CreateTestFormField("Phone", "phone", "", "tel", &field);
form.fields.push_back(field);
std::vector<FormData> forms(1, form);
base::HistogramTester histogram_tester;
autofill_manager_->OnFormsSeen(forms, TimeTicks());
// We change the value of the text fields to change the default/seen values
// (hence the values are not cleared in UpdateFromCache). The new values
// match what is in the test profile.
form.fields[1].value = base::ASCIIToUTF16("79401");
form.fields[2].value = base::ASCIIToUTF16("2345678901");
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
for (const std::string source : {"Heuristic", "Server", "Overall"}) {
std::string histogram_name =
"Autofill.FieldPredictionQuality.ByFieldType." + source;
// First verify that country was not predicted by client or server.
{
SCOPED_TRACE("ADDRESS_HOME_COUNTRY");
histogram_tester.ExpectBucketCount(
histogram_name,
GetFieldTypeGroupMetric(
ADDRESS_HOME_COUNTRY,
source == "Overall" ? AutofillMetrics::TRUE_POSITIVE
: AutofillMetrics::FALSE_NEGATIVE_UNKNOWN),
1);
}
// We did not predict zip code because it did not have an autocomplete
// attribute, nor client or server predictions.
{
SCOPED_TRACE("ADDRESS_HOME_ZIP");
histogram_tester.ExpectBucketCount(
histogram_name,
GetFieldTypeGroupMetric(ADDRESS_HOME_ZIP,
AutofillMetrics::FALSE_NEGATIVE_UNKNOWN),
1);
}
// Phone should have been predicted by the heuristics but not the server.
{
SCOPED_TRACE("PHONE_HOME_WHOLE_NUMBER");
histogram_tester.ExpectBucketCount(
histogram_name,
GetFieldTypeGroupMetric(PHONE_HOME_WHOLE_NUMBER,
source == "Server"
? AutofillMetrics::FALSE_NEGATIVE_UNKNOWN
: AutofillMetrics::TRUE_POSITIVE),
1);
}
// Sanity check.
histogram_tester.ExpectTotalCount(histogram_name, 3);
}
}
// Test that we behave sanely when the cached form differs from the submitted
// one.
TEST_F(AutofillMetricsTest, SaneMetricsWithCacheMismatch) {
// Set up our form data.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin =
url::Origin::Create(GURL("http://example_root.com/form.html"));
std::vector<ServerFieldType> heuristic_types, server_types;
FormFieldData field;
test::CreateTestFormField("Both match", "match", "Elvis Aaron Presley",
"text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FULL);
test::CreateTestFormField("Both mismatch", "mismatch", "buddy@gmail.com",
"text", &field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_NUMBER);
server_types.push_back(PHONE_HOME_NUMBER);
test::CreateTestFormField("Only heuristics match", "mixed", "Memphis", "text",
&field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(ADDRESS_HOME_CITY);
server_types.push_back(PHONE_HOME_NUMBER);
test::CreateTestFormField("Unknown", "unknown", "garbage", "text", &field);
field.is_autofilled = false;
form.fields.push_back(field);
heuristic_types.push_back(UNKNOWN_TYPE);
server_types.push_back(UNKNOWN_TYPE);
// Simulate having seen this form with the desired heuristic and server types.
// |form_structure| will be owned by |autofill_manager_|.
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
// Add a field and re-arrange the remaining form fields before submitting.
std::vector<FormFieldData> cached_fields = form.fields;
form.fields.clear();
test::CreateTestFormField("New field", "new field", "Tennessee", "text",
&field);
form.fields.push_back(field);
form.fields.push_back(cached_fields[2]);
form.fields.push_back(cached_fields[1]);
form.fields.push_back(cached_fields[3]);
form.fields.push_back(cached_fields[0]);
// Simulate form submission.
base::HistogramTester histogram_tester;
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
for (const std::string source : {"Heuristic", "Server", "Overall"}) {
std::string aggregate_histogram =
"Autofill.FieldPredictionQuality.Aggregate." + source;
std::string by_field_type_histogram =
"Autofill.FieldPredictionQuality.ByFieldType." + source;
// Unknown:
histogram_tester.ExpectBucketCount(
aggregate_histogram, AutofillMetrics::FALSE_NEGATIVE_UNKNOWN, 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(ADDRESS_HOME_STATE,
AutofillMetrics::FALSE_NEGATIVE_UNKNOWN),
1);
// Match:
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::TRUE_POSITIVE,
source == "Heuristic" ? 2 : 1);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(NAME_FULL, AutofillMetrics::TRUE_POSITIVE), 1);
// Mismatch:
histogram_tester.ExpectBucketCount(aggregate_histogram,
AutofillMetrics::FALSE_NEGATIVE_MISMATCH,
source == "Heuristic" ? 1 : 2);
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(EMAIL_ADDRESS,
AutofillMetrics::FALSE_NEGATIVE_MISMATCH),
1);
// Source dependent:
histogram_tester.ExpectBucketCount(
by_field_type_histogram,
GetFieldTypeGroupMetric(ADDRESS_HOME_CITY,
source == "Heuristic"
? AutofillMetrics::TRUE_POSITIVE
: AutofillMetrics::FALSE_NEGATIVE_MISMATCH),
1);
}
}
// Verify that when submitting an autofillable form, the stored profile metric
// is logged.
TEST_F(AutofillMetricsTest, StoredProfileCountAutofillableFormSubmission) {
// Construct a fillable form.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin =
url::Origin::Create(GURL("http://example_root.com/form.html"));
// Three fields is enough to make it an autofillable form.
FormFieldData field;
test::CreateTestFormField("Name", "name", "", "text", &field);
form.fields.push_back(field);
test::CreateTestFormField("Email", "email", "", "text", &field);
form.fields.push_back(field);
test::CreateTestFormField("Phone", "phone", "", "text", &field);
form.fields.push_back(field);
std::vector<FormData> forms(1, form);
// Simulate form submission.
base::HistogramTester histogram_tester;
autofill_manager_->OnFormsSeen(forms, TimeTicks());
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
// An autofillable form was submitted, and the number of stored profiles is
// logged.
histogram_tester.ExpectUniqueSample(
"Autofill.StoredProfileCountAtAutofillableFormSubmission", 2, 1);
}
// Verify that when submitting a non-autofillable form, the stored profile
// metric is not logged.
TEST_F(AutofillMetricsTest, StoredProfileCountNonAutofillableFormSubmission) {
base::test::ScopedFeatureList features;
features.InitAndEnableFeature(kAutofillEnforceMinRequiredFieldsForHeuristics);
// Construct a non-fillable form.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin =
url::Origin::Create(GURL("http://example_root.com/form.html"));
// Two fields is not enough to make it an autofillable form.
FormFieldData field;
test::CreateTestFormField("Name", "name", "", "text", &field);
form.fields.push_back(field);
test::CreateTestFormField("Email", "email", "", "text", &field);
form.fields.push_back(field);
std::vector<FormData> forms(1, form);
// Simulate form submission.
base::HistogramTester histogram_tester;
autofill_manager_->OnFormsSeen(forms, TimeTicks());
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
// A non-autofillable form was submitted, and number of stored profiles is NOT
// logged.
histogram_tester.ExpectTotalCount(
"Autofill.StoredProfileCountAtAutofillableFormSubmission", 0);
}
// Verify that when submitting an autofillable form, the proper number of edited
// fields is logged.
TEST_F(AutofillMetricsTest, NumberOfEditedAutofilledFields) {
// Construct a fillable form.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin =
url::Origin::Create(GURL("http://example_root.com/form.html"));
std::vector<ServerFieldType> heuristic_types, server_types;
// Three fields is enough to make it an autofillable form.
FormFieldData field;
test::CreateTestFormField("Autofilled", "autofilled", "Elvis Aaron Presley",
"text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FULL);
test::CreateTestFormField("Autofill Failed", "autofillfailed",
"buddy@gmail.com", "text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(EMAIL_ADDRESS);
server_types.push_back(EMAIL_ADDRESS);
test::CreateTestFormField("Phone", "phone", "2345678901", "tel", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
base::HistogramTester histogram_tester;
// Simulate text input in the first and second fields.
autofill_manager_->OnTextFieldDidChange(form, form.fields[0], gfx::RectF(),
TimeTicks());
autofill_manager_->OnTextFieldDidChange(form, form.fields[1], gfx::RectF(),
TimeTicks());
// Simulate form submission.
autofill_manager_->OnFormSubmitted(form, false,
SubmissionSource::FORM_SUBMISSION);
// An autofillable form was submitted, and the number of edited autofilled
// fields is logged.
histogram_tester.ExpectUniqueSample(
"Autofill.NumberOfEditedAutofilledFieldsAtSubmission", 2, 1);
}
// Verify that when resetting the autofill manager (such as during a
// navigation), the proper number of edited fields is logged.
TEST_F(AutofillMetricsTest, NumberOfEditedAutofilledFields_NoSubmission) {
// Construct a fillable form.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin = url::Origin::Create(autofill_client_.form_origin());
std::vector<ServerFieldType> heuristic_types, server_types;
// Three fields is enough to make it an autofillable form.
FormFieldData field;
test::CreateTestFormField("Autofilled", "autofilled", "Elvis Aaron Presley",
"text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(NAME_FULL);
server_types.push_back(NAME_FULL);
test::CreateTestFormField("Autofill Failed", "autofillfailed",
"buddy@gmail.com", "text", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(EMAIL_ADDRESS);
server_types.push_back(EMAIL_ADDRESS);
test::CreateTestFormField("Phone", "phone", "2345678901", "tel", &field);
field.is_autofilled = true;
form.fields.push_back(field);
heuristic_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
server_types.push_back(PHONE_HOME_CITY_AND_NUMBER);
autofill_manager_->AddSeenForm(form, heuristic_types, server_types);
base::HistogramTester histogram_tester;
// Simulate text input in the first field.
autofill_manager_->OnTextFieldDidChange(form, form.fields[0], gfx::RectF(),
TimeTicks());
// We expect metrics to be logged when the manager is reset.
autofill_manager_->Reset();
// An autofillable form was uploaded, and the number of edited autofilled
// fields is logged.
histogram_tester.ExpectUniqueSample(
"Autofill.NumberOfEditedAutofilledFieldsAtSubmission.NoSubmission", 1, 1);
}
// Verify that we correctly log metrics regarding developer engagement.
TEST_F(AutofillMetricsTest, DeveloperEngagement) {
// Start with a non-fillable form.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin = url::Origin::Create(autofill_client_.form_origin());
FormFieldData field;
test::CreateTestFormField("Name", "name", "", "text", &field);
form.fields.push_back(field);
test::CreateTestFormField("Email", "email", "", "text", &field);
form.fields.push_back(field);
std::vector<FormData> forms(1, form);
// Ensure no metrics are logged when small form support is disabled (min
// number of fields enforced).
{
base::HistogramTester histogram_tester;
autofill_manager_->OnFormsSeen(forms, TimeTicks());
autofill_manager_->Reset();
histogram_tester.ExpectTotalCount("Autofill.DeveloperEngagement", 0);
}
// Otherwise, log developer engagement for all forms.
{
base::test::ScopedFeatureList features;
features.InitAndDisableFeature(
kAutofillEnforceMinRequiredFieldsForHeuristics);
base::HistogramTester histogram_tester;
autofill_manager_->OnFormsSeen(forms, TimeTicks());
autofill_manager_->Reset();
histogram_tester.ExpectUniqueSample(
"Autofill.DeveloperEngagement",
AutofillMetrics::FILLABLE_FORM_PARSED_WITHOUT_TYPE_HINTS, 1);
}
// Add another field to the form, so that it becomes fillable.
test::CreateTestFormField("Phone", "phone", "", "text", &field);
forms.back().fields.push_back(field);
// Expect the "form parsed without hints" metric to be logged.
{
base::HistogramTester histogram_tester;
autofill_manager_->OnFormsSeen(forms, TimeTicks());
autofill_manager_->Reset();
histogram_tester.ExpectUniqueSample(
"Autofill.DeveloperEngagement",
AutofillMetrics::FILLABLE_FORM_PARSED_WITHOUT_TYPE_HINTS, 1);
}
// Add some fields with an author-specified field type to the form.
// We need to add at least three fields, because a form must have at least
// three fillable fields to be considered to be autofillable; and if at least
// one field specifies an explicit type hint, we don't apply any of our usual
// local heuristics to detect field types in the rest of the form.
test::CreateTestFormField("", "", "", "text", &field);
field.autocomplete_attribute = "given-name";
forms.back().fields.push_back(field);
test::CreateTestFormField("", "", "", "text", &field);
field.autocomplete_attribute = "email";
forms.back().fields.push_back(field);
test::CreateTestFormField("", "", "", "text", &field);
field.autocomplete_attribute = "address-line1";
forms.back().fields.push_back(field);
// Expect the "form parsed with field type hints" metric to be logged.
{
base::HistogramTester histogram_tester;
autofill_manager_->OnFormsSeen(forms, TimeTicks());
autofill_manager_->Reset();
histogram_tester.ExpectBucketCount(
"Autofill.DeveloperEngagement",
AutofillMetrics::FILLABLE_FORM_PARSED_WITH_TYPE_HINTS, 1);
histogram_tester.ExpectBucketCount(
"Autofill.DeveloperEngagement",
AutofillMetrics::FORM_CONTAINS_UPI_VPA_HINT, 0);
}
// Add a field with an author-specified UPI-VPA field type in the form.
test::CreateTestFormField("", "", "", "text", &field);
field.autocomplete_attribute = "upi-vpa";
forms.back().fields.push_back(field);
// Expect the "form parsed with type hints" metric, and the
// "author-specified upi-vpa type" metric to be logged.
{
base::HistogramTester histogram_tester;
autofill_manager_->OnFormsSeen(forms, TimeTicks());
autofill_manager_->Reset();
histogram_tester.ExpectBucketCount(
"Autofill.DeveloperEngagement",
AutofillMetrics::FILLABLE_FORM_PARSED_WITH_TYPE_HINTS, 1);
histogram_tester.ExpectBucketCount(
"Autofill.DeveloperEngagement",
AutofillMetrics::FORM_CONTAINS_UPI_VPA_HINT, 1);
}
}
// Verify that we correctly log UKM for form parsed without type hints regarding
// developer engagement.
TEST_F(AutofillMetricsTest,
UkmDeveloperEngagement_LogFillableFormParsedWithoutTypeHints) {
// Start with a non-fillable form.
FormData form;
form.name = ASCIIToUTF16("TestForm");
form.origin = GURL("http://example.com/form.html");
form.action = GURL("http://example.com/submit.html");
form.main_frame_origin = url::Origin::Create(autofill_client_.form_origin());
FormFieldData field;
test::CreateTestFormField("Name", "name", "", "text", &field);
form.fields.push_back(field);
test::CreateTestFormField("Email", "email", "", "text", &field);
form.fields.push_back(field);
std::vector<FormData> forms(1, form);