blob: 8258d594b15b62e21ade2dcb80a0871d349530ff [file] [log] [blame]
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "platform/graphics/HighContrastImageClassifier.h"
#include "base/rand_util.h"
#include "third_party/WebKit/Source/platform/geometry/IntRect.h"
#include "third_party/skia/include/utils/SkNullCanvas.h"
namespace {
bool IsColorGray(const SkColor& color) {
return abs(static_cast<int>(SkColorGetR(color)) -
static_cast<int>(SkColorGetG(color))) +
abs(static_cast<int>(SkColorGetG(color)) -
static_cast<int>(SkColorGetB(color))) <=
8;
}
void RGBAdd(const SkColor& new_pixel, SkColor4f* sink) {
sink->fR += SkColorGetR(new_pixel);
sink->fG += SkColorGetG(new_pixel);
sink->fB += SkColorGetB(new_pixel);
}
void RGBDivide(SkColor4f* sink, int divisor) {
sink->fR /= divisor;
sink->fG /= divisor;
sink->fB /= divisor;
}
float ColorDifference(const SkColor& color1, const SkColor4f& color2) {
return sqrt((pow(static_cast<float>(SkColorGetR(color1)) - color2.fR, 2.0) +
pow(static_cast<float>(SkColorGetG(color1)) - color2.fG, 2.0) +
pow(static_cast<float>(SkColorGetB(color1)) - color2.fB, 2.0)) /
3.0);
}
const int kPixelsToSample = 1000;
const int kBlocksCount1D = 10;
const int kMinImageSizeForClassification1D = 24;
} // namespace
namespace blink {
HighContrastImageClassifier::HighContrastImageClassifier()
: use_testing_random_generator_(false), testing_random_generator_seed_(0) {}
int HighContrastImageClassifier::GetRandomInt(const int min, const int max) {
if (use_testing_random_generator_) {
testing_random_generator_seed_ *= 7;
testing_random_generator_seed_ += 15485863;
testing_random_generator_seed_ %= 256205689;
return min + testing_random_generator_seed_ % (max - min);
}
return base::RandInt(min, max - 1);
}
bool HighContrastImageClassifier::ShouldApplyHighContrastFilterToImage(
Image& image) {
HighContrastClassification result = image.GetHighContrastClassification();
if (result != HighContrastClassification::kNotClassified)
return result == HighContrastClassification::kApplyHighContrastFilter;
if (image.width() < kMinImageSizeForClassification1D ||
image.height() < kMinImageSizeForClassification1D) {
result = HighContrastClassification::kApplyHighContrastFilter;
} else {
std::vector<float> features;
if (!ComputeImageFeatures(image, &features))
result = HighContrastClassification::kDoNotApplyHighContrastFilter;
else
result = ClassifyImage(features);
}
image.SetHighContrastClassification(result);
return result == HighContrastClassification::kApplyHighContrastFilter;
}
// This function computes a single feature vector based on a sample set of image
// pixels. Please refer to |GetSamples| function for description of the sampling
// method, and |GetFeatures| function for description of the features.
bool HighContrastImageClassifier::ComputeImageFeatures(
Image& image,
std::vector<float>* features) {
SkBitmap bitmap;
if (!GetBitmap(image, &bitmap))
return false;
if (use_testing_random_generator_)
testing_random_generator_seed_ = 0;
std::vector<SkColor> sampled_pixels;
float transparency_ratio;
float background_ratio;
GetSamples(bitmap, &sampled_pixels, &transparency_ratio, &background_ratio);
GetFeatures(sampled_pixels, transparency_ratio, background_ratio, features);
return true;
}
bool HighContrastImageClassifier::GetBitmap(Image& image, SkBitmap* bitmap) {
if (!image.IsBitmapImage() || !image.width() || !image.height())
return false;
bitmap->allocPixels(
SkImageInfo::MakeN32(image.width(), image.height(), kPremul_SkAlphaType));
SkCanvas canvas(*bitmap);
canvas.clear(SK_ColorTRANSPARENT);
canvas.drawImageRect(image.PaintImageForCurrentFrame().GetSkImage(),
SkRect::MakeIWH(image.width(), image.height()), nullptr);
return true;
}
// Extracts sample pixels from the image, focusing on the foreground and
// ignoring static background. The image is separated into uniformly distributed
// blocks through its width and height, each block is sampled, and checked to
// see if it seem to be background or foreground (See IsBlockBackground for
// details). Samples from foreground blocks are collected, and if they are less
// than 50% of requested samples, the foreground blocks are resampled to get
// more points.
void HighContrastImageClassifier::GetSamples(
const SkBitmap& bitmap,
std::vector<SkColor>* sampled_pixels,
float* transparency_ratio,
float* background_ratio) {
int pixels_per_block = kPixelsToSample / (kBlocksCount1D * kBlocksCount1D);
int transparent_pixels = 0;
int opaque_pixels = 0;
int blocks_count = 0;
std::vector<int> horizontal_grid(kBlocksCount1D + 1);
std::vector<int> vertical_grid(kBlocksCount1D + 1);
for (int block = 0; block <= kBlocksCount1D; block++) {
horizontal_grid[block] = static_cast<int>(
round(block * bitmap.width() / static_cast<float>(kBlocksCount1D)));
vertical_grid[block] = static_cast<int>(
round(block * bitmap.height() / static_cast<float>(kBlocksCount1D)));
}
sampled_pixels->clear();
std::vector<IntRect> foreground_blocks;
for (int y = 0; y < kBlocksCount1D; y++) {
for (int x = 0; x < kBlocksCount1D; x++) {
IntRect block(horizontal_grid[x], vertical_grid[y],
horizontal_grid[x + 1] - horizontal_grid[x],
vertical_grid[y + 1] - vertical_grid[y]);
std::vector<SkColor> block_samples;
int block_transparent_pixels;
GetBlockSamples(bitmap, block, pixels_per_block, &block_samples,
&block_transparent_pixels);
opaque_pixels += static_cast<int>(block_samples.size());
transparent_pixels += block_transparent_pixels;
if (!IsBlockBackground(block_samples, block_transparent_pixels)) {
sampled_pixels->insert(sampled_pixels->end(), block_samples.begin(),
block_samples.end());
foreground_blocks.push_back(block);
}
blocks_count++;
}
}
*transparency_ratio = static_cast<float>(transparent_pixels) /
(transparent_pixels + opaque_pixels);
*background_ratio =
1.0 - static_cast<float>(foreground_blocks.size()) / blocks_count;
// If samples are too few, resample foreground blocks.
if (sampled_pixels->size() < kPixelsToSample / 2 &&
foreground_blocks.size()) {
pixels_per_block = static_cast<int>(
ceil((kPixelsToSample - static_cast<float>(sampled_pixels->size())) /
static_cast<float>(foreground_blocks.size())));
for (const IntRect& block : foreground_blocks) {
std::vector<SkColor> block_samples;
int unused;
GetBlockSamples(bitmap, block, pixels_per_block, &block_samples, &unused);
sampled_pixels->insert(sampled_pixels->end(), block_samples.begin(),
block_samples.end());
}
}
}
// Selects random samples from a block of the image. Returns the opaque sampled
// pixels, and the number of transparent sampled pixels.
void HighContrastImageClassifier::GetBlockSamples(
const SkBitmap& bitmap,
const IntRect& block,
const int required_samples_count,
std::vector<SkColor>* sampled_pixels,
int* transparent_pixels_count) {
*transparent_pixels_count = 0;
int x1 = block.X();
int y1 = block.Y();
int x2 = block.MaxX();
int y2 = block.MaxY();
DCHECK(x1 < bitmap.width());
DCHECK(y1 < bitmap.height());
DCHECK(x2 <= bitmap.width());
DCHECK(y2 <= bitmap.height());
sampled_pixels->clear();
for (int i = 0; i < required_samples_count; i++) {
int x = GetRandomInt(x1, x2);
int y = GetRandomInt(y1, y2);
SkColor new_sample = bitmap.getColor(x, y);
if (SkColorGetA(new_sample) < 128)
(*transparent_pixels_count)++;
else
sampled_pixels->push_back(new_sample);
}
}
// Given sampled pixels and transparent pixels count of a block of the image,
// decides if that block is background or not. If more than 80% of the block is
// transparent, or the divergence of colors in the block is less than 5%, the
// block is considered background.
bool HighContrastImageClassifier::IsBlockBackground(
const std::vector<SkColor>& sampled_pixels,
const int transparent_pixels) {
if (static_cast<int>(sampled_pixels.size()) <= transparent_pixels / 4)
return true;
SkColor4f average;
average.fR = 0;
average.fG = 0;
average.fB = 0;
for (const SkColor& sample : sampled_pixels)
RGBAdd(sample, &average);
RGBDivide(&average, sampled_pixels.size());
float sum = 0;
for (const auto& sample : sampled_pixels)
sum += ColorDifference(sample, average);
sum /= 128; // Normalize to 0..1 range.
float divergence = sum / sampled_pixels.size();
return divergence < 0.05;
}
// This function computes a single feature vector from a sample set of image
// pixels. The current features are:
// 0: 1 if color, 0 if grayscale.
// 1: Ratio of the number of bucketed colors used in the image to all
// possiblities. Color buckets are represented with 4 bits per color channel.
// 2: Ratio of transparent area to the whole image.
// 3: Ratio of the brackground area to the whole image.
void HighContrastImageClassifier::GetFeatures(
const std::vector<SkColor>& sampled_pixels,
const float transparency_ratio,
const float background_ratio,
std::vector<float>* features) {
int samples_count = static_cast<int>(sampled_pixels.size());
// Is image grayscale.
int color_pixels = 0;
for (const SkColor& sample : sampled_pixels) {
if (!IsColorGray(sample))
color_pixels++;
}
ColorMode color_mode = (color_pixels > samples_count / 100)
? ColorMode::kColor
: ColorMode::kGrayscale;
features->resize(4);
// Feature 0: Is Colorful?
(*features)[0] = color_mode == ColorMode::kColor;
// Feature 1: Color Buckets Ratio.
(*features)[1] = ComputeColorBucketsRatio(sampled_pixels, color_mode);
// Feature 2: Transparency Ratio
(*features)[2] = transparency_ratio;
// Feature 3: Background Ratio.
(*features)[3] = background_ratio;
}
float HighContrastImageClassifier::ComputeColorBucketsRatio(
const std::vector<SkColor>& sampled_pixels,
const ColorMode color_mode) {
std::set<unsigned> buckets;
// If image is in color, use 4 bits per color channel, otherwise 4 bits for
// illumination.
if (color_mode == ColorMode::kColor) {
for (const SkColor& sample : sampled_pixels) {
unsigned bucket = ((SkColorGetR(sample) >> 4) << 8) +
((SkColorGetG(sample) >> 4) << 4) +
((SkColorGetB(sample) >> 4));
buckets.insert(bucket);
}
} else {
for (const SkColor& sample : sampled_pixels) {
unsigned illumination =
(SkColorGetR(sample) * 5 + SkColorGetG(sample) * 3 +
SkColorGetB(sample) * 2) /
10;
buckets.insert(illumination / 16);
}
}
// Using 4 bit per channel representation of each color bucket, there would be
// 2^4 buckets for grayscale images and 2^12 for color images.
const float max_buckets[] = {16, 4096};
return static_cast<float>(buckets.size()) /
max_buckets[color_mode == ColorMode::kColor];
}
HighContrastClassification HighContrastImageClassifier::ClassifyImage(
const std::vector<float>& features) {
bool result = false;
// Shallow decision tree trained by C4.5.
if (features.size() >= 2) {
float threshold = (features[0] == 0) ? 0.8125 : 0.0166;
result = features[1] < threshold;
}
return result ? HighContrastClassification::kApplyHighContrastFilter
: HighContrastClassification::kDoNotApplyHighContrastFilter;
}
} // namespace blink