blob: bab24d24312a9f10e85fabe8a5eccde3a64a463e [file] [log] [blame]
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "media/capabilities/learning_helper.h"
#include "base/task/post_task.h"
#include "media/learning/common/feature_library.h"
#include "media/learning/common/learning_task.h"
namespace media {
using learning::FeatureLibrary;
using learning::FeatureProviderFactoryCB;
using learning::FeatureValue;
using learning::LabelledExample;
using learning::LearningSessionImpl;
using learning::LearningTask;
using learning::LearningTaskController;
using learning::ObservationCompletion;
using learning::SequenceBoundFeatureProvider;
using learning::TargetValue;
// Remember that these are used to construct UMA histogram names! Be sure to
// update histograms.xml if you change them!
// Dropped frame ratio, default features, regression tree.
const char* const kDroppedFrameRatioBaseTreeTaskName = "BaseTree";
// Dropped frame ratio, default+FeatureLibrary features, regression tree.
const char* const kDroppedFrameRatioEnhancedTreeTaskName = "EnhancedTree";
// Dropped frame ratio, default+FeatureLibrary features, regression tree,
// examples are unweighted.
const char* const kDroppedFrameRatioEnhancedUnweightedTreeTaskName =
// Binary smoothness, default+FeatureLibrary features, regression tree,
// examples are unweighted.
const char* const kBinarySmoothnessEnhancedUnweightedTreeTaskName =
// Dropped frame ratio, default features, lookup table.
const char* const kDroppedFrameRatioBaseTableTaskName = "BaseTable";
// Threshold for the dropped frame to total frame ratio, at which we'll decide
// that the playback was not smooth.
constexpr double kSmoothnessThreshold = 0.1;
LearningHelper::LearningHelper(FeatureProviderFactoryCB feature_factory) {
// Create the LearningSession on a background task runner. In the future,
// it's likely that the session will live on the main thread, and handle
// delegation of LearningTaskControllers to other threads. However, for now,
// do it here.
learning_session_ = std::make_unique<LearningSessionImpl>(
// Register a few learning tasks.
// We only do this here since we own the session. Normally, whatever creates
// the session would register all the learning tasks.
LearningTask dropped_frame_task(
kDroppedFrameRatioBaseTableTaskName, LearningTask::Model::kLookupTable,
{"width", ::media::learning::LearningTask::Ordering::kNumeric},
{"height", ::media::learning::LearningTask::Ordering::kNumeric},
{"frame_rate", ::media::learning::LearningTask::Ordering::kNumeric},
{"dropped_ratio", LearningTask::Ordering::kNumeric}));
// Report results hackily both in aggregate and by training data weight.
dropped_frame_task.smoothness_threshold = kSmoothnessThreshold;
dropped_frame_task.uma_hacky_aggregate_confusion_matrix = true;
dropped_frame_task.uma_hacky_by_training_weight_confusion_matrix = true;
// Pick a max reporting weight that represents the total number of frames.
// This will record in bucket [0, 4999], [5000, 9999], etc. Unlike the
// existing mcap thresholds, these are not per-bucket. That's why they're 10x
// higher than the per-bucket thresholds we're using there. Mcap allows on
// the order of 2,500 frames in each of {resolution X fps X codec} buckets,
// while the reported training weight here would be total for the whole set.
// So, we multiply by about 20 to approximate the number of buckets to keep
// it about the same as the size of the cross product.
const double weighted_reporting_max = 49999.;
dropped_frame_task.max_reporting_weight = weighted_reporting_max;
base_table_controller_ =
// Modify the task to use ExtraTrees. = kDroppedFrameRatioBaseTreeTaskName;
dropped_frame_task.model = LearningTask::Model::kExtraTrees;
base_tree_controller_ =
// Add common features, if we have a factory.
if (feature_factory) { = kDroppedFrameRatioEnhancedTreeTaskName;
{"origin", ::media::learning::LearningTask::Ordering::kUnordered});
enhanced_tree_controller_ =
// Duplicate the task with a new name and UMA histogram. We'll add
// unweighted examples to it to see which one does better. = kDroppedFrameRatioEnhancedUnweightedTreeTaskName;
// Adjust the reporting weight since we'll have 100 or fewer examples.
dropped_frame_task.max_reporting_weight = 99.;
unweighted_tree_controller_ =
// Set up the binary smoothness task. This has a nominal target, with
// "smooth" as 0, and "not smooth" as 1. This is so that the low numbers
// are still smooth, and the hight numbers are still not smooth. It makes
// reporting the same for both. = kBinarySmoothnessEnhancedUnweightedTreeTaskName;
/* TODO(liberato): DistributionReporter only supports regression, so we
leave it as kNumeric. Since we only add 0,1 as targets, it's probably
fairly close to the same thing.
dropped_frame_task.target_description = {
"is_smooth", ::media::learning::LearningTask::Ordering::kUnordered};
// We'll threshold the ratio when figuring out the binary label, so we just
// want to pick the majority. Note that I have no idea if this is actually
// the best threshold, but it seems like a good place to start.
dropped_frame_task.smoothness_threshold = 0.5;
dropped_frame_task.max_reporting_weight = weighted_reporting_max;
binary_tree_controller_ =
LearningHelper::~LearningHelper() = default;
void LearningHelper::AppendStats(
const VideoDecodeStatsDB::VideoDescKey& video_key,
learning::FeatureValue origin,
const VideoDecodeStatsDB::DecodeStatsEntry& new_stats) {
// If no frames were recorded, then do nothing.
if (new_stats.frames_decoded == 0)
// Sanity.
if (new_stats.frames_dropped > new_stats.frames_decoded)
// Add a training example for |new_stats|.
LabelledExample example;
// Extract features from |video_key|.
// Record the ratio of dropped frames to non-dropped frames. Weight this
// example by the total number of frames, since we want to predict the
// aggregate dropped frames ratio. That lets us compare with the current
// implementation directly.
// It's also not clear that we want to do this; we might want to weight each
// playback equally and predict the dropped frame ratio. For example, if
// there is a dependence on video length, then it's unclear that weighting
// the examples is the right thing to do.
example.target_value = TargetValue(
static_cast<double>(new_stats.frames_dropped) / new_stats.frames_decoded);
example.weight = new_stats.frames_decoded;
// Add this example to all tasks.
AddExample(base_table_controller_.get(), example);
AddExample(base_tree_controller_.get(), example);
if (enhanced_tree_controller_) {
AddExample(enhanced_tree_controller_.get(), example);
// Also add to the unweighted model.
example.weight = 1u;
AddExample(unweighted_tree_controller_.get(), example);
// Threshold the target to 0 for "smooth", and 1 for "not smooth".
example.target_value =
TargetValue(example.target_value.value() > kSmoothnessThreshold);
AddExample(binary_tree_controller_.get(), example);
void LearningHelper::AddExample(LearningTaskController* controller,
const LabelledExample& example) {
base::UnguessableToken id = base::UnguessableToken::Create();
controller->BeginObservation(id, example.features);
id, ObservationCompletion(example.target_value, example.weight));
} // namespace media