chrome/renderer/safe_browsing/phishing_url_feature_extractor.cc

// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/renderer/safe_browsing/phishing_url_feature_extractor.h"

#include <algorithm>
#include <string>
#include <vector>

#include "base/logging.h"
#include "base/metrics/histogram_macros.h"
#include "base/strings/string_split.h"
#include "base/strings/string_util.h"
#include "base/timer/elapsed_timer.h"
#include "chrome/renderer/safe_browsing/features.h"
#include "net/base/registry_controlled_domains/registry_controlled_domain.h"
#include "url/gurl.h"

namespace safe_browsing {

PhishingUrlFeatureExtractor::PhishingUrlFeatureExtractor() {}

PhishingUrlFeatureExtractor::~PhishingUrlFeatureExtractor() {}

bool PhishingUrlFeatureExtractor::ExtractFeatures(const GURL& url,
                                                  FeatureMap* features) {
  base::ElapsedTimer timer;
  if (url.HostIsIPAddress()) {
    if (!features->AddBooleanFeature(features::kUrlHostIsIpAddress))
      return false;
  } else {
    // Remove any leading/trailing dots.
    std::string host;
    base::TrimString(url.host(), ".", &host);

    // TODO(bryner): Ensure that the url encoding is consistent with
    // the features in the model.

    // Disallow unknown registries so that we don't classify
    // partial hostnames (e.g. "www.subdomain").
    size_t registry_length =
        net::registry_controlled_domains::GetCanonicalHostRegistryLength(
            host, net::registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES,
            net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES);

    if (registry_length == 0 || registry_length == std::string::npos) {
      DVLOG(1) << "Could not find TLD for host: " << host;
      return false;
    }
    DCHECK_LT(registry_length, host.size()) << "Non-zero registry length, but "
        "host is only a TLD: " << host;
    size_t tld_start = host.size() - registry_length;
    if (!features->AddBooleanFeature(features::kUrlTldToken +
                                     host.substr(tld_start)))
      return false;

    // Pull off the TLD and the preceeding dot.
    host.erase(tld_start - 1);
    std::vector<std::string> host_tokens = base::SplitString(
        host, ".", base::KEEP_WHITESPACE, base::SPLIT_WANT_NONEMPTY);
    if (host_tokens.empty()) {
      DVLOG(1) << "Could not find domain for host: " << host;
      return false;
    }
    if (!features->AddBooleanFeature(features::kUrlDomainToken +
                                     host_tokens.back()))
      return false;
    host_tokens.pop_back();

    // Now we're just left with the "other" host tokens.
    for (auto it = host_tokens.begin(); it != host_tokens.end(); ++it) {
      if (!features->AddBooleanFeature(features::kUrlOtherHostToken + *it))
        return false;
    }

    if (host_tokens.size() > 1) {
      if (!features->AddBooleanFeature(features::kUrlNumOtherHostTokensGTOne))
        return false;
      if (host_tokens.size() > 3) {
        if (!features->AddBooleanFeature(
                features::kUrlNumOtherHostTokensGTThree))
          return false;
      }
    }
  }

  std::vector<std::string> long_tokens;
  SplitStringIntoLongAlphanumTokens(url.path(), &long_tokens);
  for (const std::string& token : long_tokens) {
    if (!features->AddBooleanFeature(features::kUrlPathToken + token))
      return false;
  }

  UMA_HISTOGRAM_TIMES("SBClientPhishing.URLFeatureTime", timer.Elapsed());
  return true;
}

// static
void PhishingUrlFeatureExtractor::SplitStringIntoLongAlphanumTokens(
    const std::string& full,
    std::vector<std::string>* tokens) {
  // Split on common non-alphanumerics.
  // TODO(bryner): Split on all(?) non-alphanumerics and handle %XX properly.
  static const char kTokenSeparators[] = ".,\\/_-|=%:!&";
  for (const base::StringPiece& token :
       base::SplitStringPiece(full, kTokenSeparators, base::KEEP_WHITESPACE,
                              base::SPLIT_WANT_NONEMPTY)) {
    // Copy over only the splits that are 3 or more chars long.
    // TODO(bryner): Determine a meaningful min size.
    if (token.length() >= kMinPathComponentLength)
      tokens->push_back(token.as_string());
  }
}

}  // namespace safe_browsing