blob: 6c6b300ffb67a5c1f315d649a76278007442c83c [file] [log] [blame]
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/safe_browsing/db/v4_protocol_manager_util.h"
#include "base/base64.h"
#include "base/hash.h"
#include "base/metrics/histogram_functions.h"
#include "base/rand_util.h"
#include "base/sha1.h"
#include "base/strings/string_util.h"
#include "base/strings/stringprintf.h"
#include "crypto/sha2.h"
#include "net/base/escape.h"
#include "net/base/ip_address.h"
#include "net/http/http_request_headers.h"
#include "url/url_util.h"
using base::Time;
using base::TimeDelta;
namespace safe_browsing {
// Can be overriden by tests.
const char* g_sbv4_url_prefix_for_testing = nullptr;
const char kSbV4UrlPrefix[] = "https://safebrowsing.googleapis.com/v4";
const base::FilePath::CharType kStoreSuffix[] = FILE_PATH_LITERAL(".store");
namespace {
std::string Unescape(const std::string& url) {
std::string unescaped_str(url);
const int kMaxLoopIterations = 1024;
size_t old_size = 0;
int loop_var = 0;
do {
old_size = unescaped_str.size();
unescaped_str = net::UnescapeBinaryURLComponent(unescaped_str);
} while (old_size != unescaped_str.size() &&
++loop_var <= kMaxLoopIterations);
return unescaped_str;
}
std::string Escape(const std::string& url) {
std::string escaped_str;
// The escaped string is larger so allocate double the length to reduce the
// chance of the string being grown.
escaped_str.reserve(url.length() * 2);
const char* kHexString = "0123456789ABCDEF";
for (size_t i = 0; i < url.length(); i++) {
unsigned char c = static_cast<unsigned char>(url[i]);
if (c <= ' ' || c > '~' || c == '#' || c == '%') {
escaped_str += '%';
escaped_str += kHexString[c >> 4];
escaped_str += kHexString[c & 0xf];
} else {
escaped_str += c;
}
}
return escaped_str;
}
} // namespace
void SetSbV4UrlPrefixForTesting(const char* url_prefix) {
g_sbv4_url_prefix_for_testing = url_prefix;
}
std::ostream& operator<<(std::ostream& os, const ListIdentifier& id) {
os << "{hash: " << id.hash() << "; platform_type: " << id.platform_type()
<< "; threat_entry_type: " << id.threat_entry_type()
<< "; threat_type: " << id.threat_type() << "}";
return os;
}
PlatformType GetCurrentPlatformType() {
#if defined(OS_WIN)
return WINDOWS_PLATFORM;
#elif defined(OS_LINUX)
return LINUX_PLATFORM;
#elif defined(OS_MACOSX)
return OSX_PLATFORM;
#else
// This should ideally never compile but it is getting compiled on Android.
// See: https://bugs.chromium.org/p/chromium/issues/detail?id=621647
// TODO(vakh): Once that bug is fixed, this should be removed. If we leave
// the platform_type empty, the server won't recognize the request and
// return an error response which will pollute our UMA metrics.
return LINUX_PLATFORM;
#endif
}
ListIdentifier GetCertCsdDownloadWhitelistId() {
return ListIdentifier(GetCurrentPlatformType(), CERT, CSD_DOWNLOAD_WHITELIST);
}
ListIdentifier GetChromeExtMalwareId() {
return ListIdentifier(CHROME_PLATFORM, CHROME_EXTENSION, MALWARE_THREAT);
}
ListIdentifier GetChromeUrlApiId() {
return ListIdentifier(CHROME_PLATFORM, URL, API_ABUSE);
}
ListIdentifier GetChromeUrlClientIncidentId() {
return ListIdentifier(CHROME_PLATFORM, URL, CLIENT_INCIDENT);
}
ListIdentifier GetIpMalwareId() {
return ListIdentifier(GetCurrentPlatformType(), IP_RANGE, MALWARE_THREAT);
}
ListIdentifier GetUrlCsdDownloadWhitelistId() {
return ListIdentifier(GetCurrentPlatformType(), URL, CSD_DOWNLOAD_WHITELIST);
}
ListIdentifier GetUrlCsdWhitelistId() {
return ListIdentifier(GetCurrentPlatformType(), URL, CSD_WHITELIST);
}
ListIdentifier GetUrlMalwareId() {
return ListIdentifier(GetCurrentPlatformType(), URL, MALWARE_THREAT);
}
ListIdentifier GetUrlMalBinId() {
return ListIdentifier(GetCurrentPlatformType(), URL, MALICIOUS_BINARY);
}
ListIdentifier GetUrlSocEngId() {
return ListIdentifier(GetCurrentPlatformType(), URL, SOCIAL_ENGINEERING);
}
ListIdentifier GetUrlSubresourceFilterId() {
return ListIdentifier(GetCurrentPlatformType(), URL, SUBRESOURCE_FILTER);
}
ListIdentifier GetUrlSuspiciousSiteId() {
return ListIdentifier(GetCurrentPlatformType(), URL, SUSPICIOUS);
}
ListIdentifier GetUrlUwsId() {
return ListIdentifier(GetCurrentPlatformType(), URL, UNWANTED_SOFTWARE);
}
std::string GetUmaSuffixForStore(const base::FilePath& file_path) {
DCHECK_EQ(kStoreSuffix, file_path.BaseName().Extension());
return base::StringPrintf(
".%" PRIsFP, file_path.BaseName().RemoveExtension().value().c_str());
}
StoreAndHashPrefix::StoreAndHashPrefix(ListIdentifier list_id,
const HashPrefix& hash_prefix)
: list_id(list_id), hash_prefix(hash_prefix) {}
StoreAndHashPrefix::~StoreAndHashPrefix() {}
bool StoreAndHashPrefix::operator==(const StoreAndHashPrefix& other) const {
return list_id == other.list_id && hash_prefix == other.hash_prefix;
}
bool StoreAndHashPrefix::operator!=(const StoreAndHashPrefix& other) const {
return !operator==(other);
}
size_t StoreAndHashPrefix::hash() const {
std::size_t first = list_id.hash();
std::size_t second = std::hash<std::string>()(hash_prefix);
return base::HashInts(first, second);
}
bool SBThreatTypeSetIsValidForCheckBrowseUrl(const SBThreatTypeSet& set) {
for (SBThreatType type : set) {
switch (type) {
case SB_THREAT_TYPE_URL_PHISHING:
case SB_THREAT_TYPE_URL_MALWARE:
case SB_THREAT_TYPE_URL_UNWANTED:
case SB_THREAT_TYPE_SUSPICIOUS_SITE:
break;
default:
return false;
}
}
return true;
}
bool ListIdentifier::operator==(const ListIdentifier& other) const {
return platform_type_ == other.platform_type_ &&
threat_entry_type_ == other.threat_entry_type_ &&
threat_type_ == other.threat_type_;
}
bool ListIdentifier::operator!=(const ListIdentifier& other) const {
return !operator==(other);
}
size_t ListIdentifier::hash() const {
std::size_t first = std::hash<unsigned int>()(platform_type_);
std::size_t second = std::hash<unsigned int>()(threat_entry_type_);
std::size_t third = std::hash<unsigned int>()(threat_type_);
std::size_t interim = base::HashInts(first, second);
return base::HashInts(interim, third);
}
ListIdentifier::ListIdentifier(PlatformType platform_type,
ThreatEntryType threat_entry_type,
ThreatType threat_type)
: platform_type_(platform_type),
threat_entry_type_(threat_entry_type),
threat_type_(threat_type) {
DCHECK(PlatformType_IsValid(platform_type));
DCHECK(ThreatEntryType_IsValid(threat_entry_type));
DCHECK(ThreatType_IsValid(threat_type));
}
ListIdentifier::ListIdentifier(const ListUpdateResponse& response)
: ListIdentifier(response.platform_type(),
response.threat_entry_type(),
response.threat_type()) {}
V4ProtocolConfig::V4ProtocolConfig(const std::string& client_name,
bool disable_auto_update,
const std::string& key_param,
const std::string& version)
: client_name(client_name),
disable_auto_update(disable_auto_update),
key_param(key_param),
version(version) {}
V4ProtocolConfig::V4ProtocolConfig(const V4ProtocolConfig& other) = default;
V4ProtocolConfig::~V4ProtocolConfig() {}
// static
base::TimeDelta V4ProtocolManagerUtil::GetNextBackOffInterval(
size_t* error_count,
size_t* multiplier) {
DCHECK(multiplier && error_count);
(*error_count)++;
if (*error_count > 1 && *error_count < 9) {
// With error count 9 and above we will hit the 24 hour max interval.
// Cap the multiplier here to prevent integer overflow errors.
*multiplier *= 2;
}
base::TimeDelta next =
base::TimeDelta::FromMinutes(*multiplier * (1 + base::RandDouble()) * 15);
base::TimeDelta day = base::TimeDelta::FromHours(24);
return next < day ? next : day;
}
// static
void V4ProtocolManagerUtil::RecordHttpResponseOrErrorCode(
const char* metric_name,
int net_error,
int response_code) {
base::UmaHistogramSparse(metric_name,
net_error == net::OK ? response_code : net_error);
}
// static
void V4ProtocolManagerUtil::GetRequestUrlAndHeaders(
const std::string& request_base64,
const std::string& method_name,
const V4ProtocolConfig& config,
GURL* gurl,
net::HttpRequestHeaders* headers) {
const char* url_prefix = g_sbv4_url_prefix_for_testing
? g_sbv4_url_prefix_for_testing
: kSbV4UrlPrefix;
*gurl = GURL(
ComposeUrl(url_prefix, method_name, request_base64, config.key_param));
UpdateHeaders(headers);
}
// static
std::string V4ProtocolManagerUtil::ComposeUrl(const std::string& prefix,
const std::string& method,
const std::string& request_base64,
const std::string& key_param) {
DCHECK(!prefix.empty() && !method.empty());
std::string url = base::StringPrintf(
"%s/%s?$req=%s&$ct=application/x-protobuf", prefix.c_str(),
method.c_str(), request_base64.c_str());
if (!key_param.empty()) {
base::StringAppendF(&url, "&key=%s",
net::EscapeQueryParamValue(key_param, true).c_str());
}
return url;
}
// static
void V4ProtocolManagerUtil::UpdateHeaders(net::HttpRequestHeaders* headers) {
// NOTE(vakh): The following header informs the envelope server (which sits in
// front of Google's stubby server) that the received GET request should be
// interpreted as a POST.
headers->SetHeaderIfMissing("X-HTTP-Method-Override", "POST");
}
// static
void V4ProtocolManagerUtil::UrlToFullHashes(
const GURL& url,
std::vector<FullHash>* full_hashes) {
std::string canon_host, canon_path, canon_query;
CanonicalizeUrl(url, &canon_host, &canon_path, &canon_query);
std::vector<std::string> hosts;
if (url.HostIsIPAddress()) {
hosts.push_back(url.host());
} else {
GenerateHostVariantsToCheck(canon_host, &hosts);
}
std::vector<std::string> paths;
GeneratePathVariantsToCheck(canon_path, canon_query, &paths);
for (const std::string& host : hosts) {
for (const std::string& path : paths) {
full_hashes->push_back(crypto::SHA256HashString(host + path));
}
}
}
// static
bool V4ProtocolManagerUtil::FullHashToHashPrefix(const FullHash& full_hash,
PrefixSize prefix_size,
HashPrefix* hash_prefix) {
if (full_hash.size() < prefix_size) {
return false;
}
*hash_prefix = full_hash.substr(0, prefix_size);
return true;
}
// static
bool V4ProtocolManagerUtil::FullHashToSmallestHashPrefix(
const FullHash& full_hash,
HashPrefix* hash_prefix) {
return FullHashToHashPrefix(full_hash, kMinHashPrefixLength, hash_prefix);
}
// static
bool V4ProtocolManagerUtil::FullHashMatchesHashPrefix(
const FullHash& full_hash,
const HashPrefix& hash_prefix) {
return full_hash.compare(0, hash_prefix.length(), hash_prefix) == 0;
}
// static
void V4ProtocolManagerUtil::GenerateHostsToCheck(
const GURL& url,
std::vector<std::string>* hosts) {
std::string canon_host;
CanonicalizeUrl(url, &canon_host, nullptr, nullptr);
GenerateHostVariantsToCheck(canon_host, hosts);
}
// static
void V4ProtocolManagerUtil::GeneratePathsToCheck(
const GURL& url,
std::vector<std::string>* paths) {
std::string canon_path;
std::string canon_query;
CanonicalizeUrl(url, nullptr, &canon_path, &canon_query);
GeneratePathVariantsToCheck(canon_path, canon_query, paths);
}
// static
void V4ProtocolManagerUtil::GeneratePatternsToCheck(
const GURL& url,
std::vector<std::string>* urls) {
std::string canon_host;
std::string canon_path;
std::string canon_query;
CanonicalizeUrl(url, &canon_host, &canon_path, &canon_query);
std::vector<std::string> hosts, paths;
GenerateHostVariantsToCheck(canon_host, &hosts);
GeneratePathVariantsToCheck(canon_path, canon_query, &paths);
for (size_t h = 0; h < hosts.size(); ++h) {
for (size_t p = 0; p < paths.size(); ++p) {
urls->push_back(hosts[h] + paths[p]);
}
}
}
// static
void V4ProtocolManagerUtil::CanonicalizeUrl(const GURL& url,
std::string* canonicalized_hostname,
std::string* canonicalized_path,
std::string* canonicalized_query) {
DCHECK(url.is_valid());
// We only canonicalize "normal" URLs.
if (!url.IsStandard())
return;
// Following canonicalization steps are excluded since url parsing takes care
// of those :-
// 1. Remove any tab (0x09), CR (0x0d), and LF (0x0a) chars from url.
// (Exclude escaped version of these chars).
// 2. Normalize hostname to 4 dot-seperated decimal values.
// 3. Lowercase hostname.
// 4. Resolve path sequences "/../" and "/./".
// That leaves us with the following :-
// 1. Remove fragment in URL.
GURL url_without_fragment;
GURL::Replacements f_replacements;
f_replacements.ClearRef();
f_replacements.ClearUsername();
f_replacements.ClearPassword();
url_without_fragment = url.ReplaceComponents(f_replacements);
// 2. Do URL unescaping until no more hex encoded characters exist.
std::string url_unescaped_str(Unescape(url_without_fragment.spec()));
url::Parsed parsed;
url::ParseStandardURL(url_unescaped_str.data(), url_unescaped_str.length(),
&parsed);
// 3. In hostname, remove all leading and trailing dots.
base::StringPiece host;
if (parsed.host.len > 0)
host.set(url_unescaped_str.data() + parsed.host.begin, parsed.host.len);
base::StringPiece host_without_end_dots =
base::TrimString(host, ".", base::TrimPositions::TRIM_ALL);
// 4. In hostname, replace consecutive dots with a single dot.
std::string host_without_consecutive_dots(
RemoveConsecutiveChars(host_without_end_dots, '.'));
// 5. In path, replace runs of consecutive slashes with a single slash.
base::StringPiece path;
if (parsed.path.len > 0)
path.set(url_unescaped_str.data() + parsed.path.begin, parsed.path.len);
std::string path_without_consecutive_slash(RemoveConsecutiveChars(path, '/'));
url::Replacements<char> hp_replacements;
hp_replacements.SetHost(
host_without_consecutive_dots.data(),
url::Component(0, host_without_consecutive_dots.length()));
hp_replacements.SetPath(
path_without_consecutive_slash.data(),
url::Component(0, path_without_consecutive_slash.length()));
std::string url_unescaped_with_can_hostpath;
url::StdStringCanonOutput output(&url_unescaped_with_can_hostpath);
url::Parsed temp_parsed;
url::ReplaceComponents(url_unescaped_str.data(), url_unescaped_str.length(),
parsed, hp_replacements, nullptr, &output,
&temp_parsed);
output.Complete();
// 6. Step needed to revert escaping done in url::ReplaceComponents.
url_unescaped_with_can_hostpath = Unescape(url_unescaped_with_can_hostpath);
// 7. After performing all above steps, percent-escape all chars in url which
// are <= ASCII 32, >= 127, #, %. Escapes must be uppercase hex characters.
std::string escaped_canon_url_str(Escape(url_unescaped_with_can_hostpath));
url::Parsed final_parsed;
url::ParseStandardURL(escaped_canon_url_str.data(),
escaped_canon_url_str.length(), &final_parsed);
if (canonicalized_hostname && final_parsed.host.len > 0) {
*canonicalized_hostname = escaped_canon_url_str.substr(
final_parsed.host.begin, final_parsed.host.len);
}
if (canonicalized_path && final_parsed.path.len > 0) {
*canonicalized_path = escaped_canon_url_str.substr(final_parsed.path.begin,
final_parsed.path.len);
}
if (canonicalized_query && final_parsed.query.len > 0) {
*canonicalized_query = escaped_canon_url_str.substr(
final_parsed.query.begin, final_parsed.query.len);
}
}
// static
std::string V4ProtocolManagerUtil::RemoveConsecutiveChars(base::StringPiece str,
const char c) {
std::string output;
// Output is at most the length of the original string.
output.reserve(str.size());
size_t i = 0;
while (i < str.size()) {
output.append(1, str[i++]);
if (str[i - 1] == c) {
while (i < str.size() && str[i] == c) {
i++;
}
}
}
return output;
}
// static
void V4ProtocolManagerUtil::GenerateHostVariantsToCheck(
const std::string& host,
std::vector<std::string>* hosts) {
hosts->clear();
if (host.empty())
return;
// Per the Safe Browsing Protocol v2 spec, we try the host, and also up to 4
// hostnames formed by starting with the last 5 components and successively
// removing the leading component. The last component isn't examined alone,
// since it's the TLD or a subcomponent thereof.
//
// Note that we don't need to be clever about stopping at the "real" eTLD --
// the data on the server side has been filtered to ensure it will not
// blacklist a whole TLD, and it's not significantly slower on our side to
// just check too much.
//
// Also note that because we have a simple blacklist, not some sort of complex
// whitelist-in-blacklist or vice versa, it doesn't matter what order we check
// these in.
const size_t kMaxHostsToCheck = 4;
bool skipped_last_component = false;
for (std::string::const_reverse_iterator i(host.rbegin());
i != host.rend() && hosts->size() < kMaxHostsToCheck; ++i) {
if (*i == '.') {
if (skipped_last_component)
hosts->push_back(std::string(i.base(), host.end()));
else
skipped_last_component = true;
}
}
hosts->push_back(host);
}
// static
void V4ProtocolManagerUtil::GeneratePathVariantsToCheck(
const std::string& path,
const std::string& query,
std::vector<std::string>* paths) {
paths->clear();
if (path.empty())
return;
// Per the Safe Browsing Protocol v2 spec, we try the exact path with/without
// the query parameters, and also up to 4 paths formed by starting at the root
// and adding more path components.
//
// As with the hosts above, it doesn't matter what order we check these in.
const size_t kMaxPathsToCheck = 4;
for (std::string::const_iterator i(path.begin());
i != path.end() && paths->size() < kMaxPathsToCheck; ++i) {
if (*i == '/')
paths->push_back(std::string(path.begin(), i + 1));
}
if (!paths->empty() && paths->back() != path)
paths->push_back(path);
if (!query.empty())
paths->push_back(path + "?" + query);
}
// static
void V4ProtocolManagerUtil::SetClientInfoFromConfig(
ClientInfo* client_info,
const V4ProtocolConfig& config) {
DCHECK(client_info);
client_info->set_client_id(config.client_name);
client_info->set_client_version(config.version);
}
// static
bool V4ProtocolManagerUtil::GetIPV6AddressFromString(
const std::string& ip_address,
net::IPAddress* address) {
DCHECK(address);
if (!address->AssignFromIPLiteral(ip_address))
return false;
if (address->IsIPv4())
*address = net::ConvertIPv4ToIPv4MappedIPv6(*address);
return address->IsIPv6();
}
// static
bool V4ProtocolManagerUtil::IPAddressToEncodedIPV6Hash(
const std::string& ip_address,
FullHash* hashed_encoded_ip) {
net::IPAddress address;
if (!GetIPV6AddressFromString(ip_address, &address)) {
return false;
}
std::string packed_ip = net::IPAddressToPackedString(address);
if (packed_ip.empty()) {
return false;
}
const std::string hash = base::SHA1HashString(packed_ip);
DCHECK_EQ(20u, hash.size());
hashed_encoded_ip->resize(hash.size() + 1);
hashed_encoded_ip->replace(0, hash.size(), hash);
(*hashed_encoded_ip)[hash.size()] = static_cast<unsigned char>(128);
return true;
}
// static
void V4ProtocolManagerUtil::GetListClientStatesFromStoreStateMap(
const std::unique_ptr<StoreStateMap>& store_state_map,
std::vector<std::string>* list_client_states) {
std::transform(
store_state_map->begin(), store_state_map->end(),
std::back_inserter(*list_client_states),
[](const std::map<ListIdentifier, std::string>::value_type& pair) {
return pair.second;
});
}
} // namespace safe_browsing