chrome/browser/safe_browsing/browser_feature_extractor.cc - chromium/src - Git at Google

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 #include "chrome/browser/safe_browsing/browser_feature_extractor.h"

 #include <stddef.h>

 #include <map>
 #include <utility>

 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/format_macros.h"
 #include "base/location.h"
 #include "base/single_thread_task_runner.h"
 #include "base/stl_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/time/time.h"
 #include "chrome/browser/history/history_service_factory.h"
 #include "chrome/browser/profiles/profile.h"
 #include "chrome/browser/safe_browsing/browser_features.h"
 #include "chrome/browser/safe_browsing/client_side_detection_host.h"
 #include "components/history/core/browser/history_service.h"
 #include "components/history/core/browser/history_types.h"
 #include "components/safe_browsing/db/database_manager.h"
 #include "components/safe_browsing/proto/csd.pb.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/navigation_controller.h"
 #include "content/public/browser/navigation_entry.h"
 #include "content/public/browser/web_contents.h"
 #include "ui/base/page_transition_types.h"
 #include "url/gurl.h"

 using content::BrowserThread;
 using content::NavigationController;
 using content::NavigationEntry;
 using content::ResourceType;
 using content::WebContents;

 namespace safe_browsing {

 namespace {

 const int kMaxMalwareIPPerRequest = 5;

 void FilterBenignIpsOnIOThread(
     scoped_refptr<SafeBrowsingDatabaseManager> database_manager,
     IPUrlMap* ips) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   for (IPUrlMap::iterator it = ips->begin(); it != ips->end();) {
     if (!database_manager.get() ||
         !database_manager->MatchMalwareIP(it->first)) {
       // it++ here returns a copy of the old iterator and passes it to erase.
       ips->erase(it++);
     } else {
       ++it;
     }
   }
 }
 }  // namespace

 IPUrlInfo::IPUrlInfo(const std::string& url,
                      const std::string& method,
                      const std::string& referrer,
                      const ResourceType& resource_type)
       : url(url),
         method(method),
         referrer(referrer),
         resource_type(resource_type) {
 }

 IPUrlInfo::IPUrlInfo(const IPUrlInfo& other) = default;

 IPUrlInfo::~IPUrlInfo() {}

 BrowseInfo::BrowseInfo() : http_status_code(0) {}

 BrowseInfo::~BrowseInfo() {}

 static void AddFeature(const std::string& feature_name,
                        double feature_value,
                        ClientPhishingRequest* request) {
   DCHECK(request);
   ClientPhishingRequest::Feature* feature =
       request->add_non_model_feature_map();
   feature->set_name(feature_name);
   feature->set_value(feature_value);
   DVLOG(2) << "Browser feature: " << feature->name() << " " << feature->value();
 }

 static void AddMalwareIpUrlInfo(const std::string& ip,
                                 const std::vector<IPUrlInfo>& meta_infos,
                                 ClientMalwareRequest* request) {
   DCHECK(request);
   for (std::vector<IPUrlInfo>::const_iterator it = meta_infos.begin();
        it != meta_infos.end(); ++it) {
     ClientMalwareRequest::UrlInfo* urlinfo =
         request->add_bad_ip_url_info();
     // We add the information about url on the bad ip.
     urlinfo->set_ip(ip);
     urlinfo->set_url(it->url);
     urlinfo->set_method(it->method);
     urlinfo->set_referrer(it->referrer);
     urlinfo->set_resource_type(static_cast<int>(it->resource_type));
   }
   DVLOG(2) << "Added url info for bad ip: " << ip;
 }

 static void AddNavigationFeatures(
     const std::string& feature_prefix,
     const NavigationController& controller,
     int index,
     const std::vector<GURL>& redirect_chain,
     ClientPhishingRequest* request) {
   NavigationEntry* entry = controller.GetEntryAtIndex(index);
   bool is_secure_referrer = entry->GetReferrer().url.SchemeIsCryptographic();
   if (!is_secure_referrer) {
     AddFeature(base::StringPrintf("%s%s=%s",
                                   feature_prefix.c_str(),
                                   features::kReferrer,
                                   entry->GetReferrer().url.spec().c_str()),
                1.0,
                request);
   }
   AddFeature(feature_prefix + features::kHasSSLReferrer,
              is_secure_referrer ? 1.0 : 0.0,
              request);
   AddFeature(feature_prefix + features::kPageTransitionType,
              static_cast<double>(
                  ui::PageTransitionStripQualifier(
                     entry->GetTransitionType())),
              request);
   AddFeature(feature_prefix + features::kIsFirstNavigation,
              index == 0 ? 1.0 : 0.0,
              request);
   // Redirect chain should always be at least of size one, as the rendered
   // url is the last element in the chain.
   if (redirect_chain.empty()) {
     NOTREACHED();
     return;
   }
   if (redirect_chain.back() != entry->GetURL()) {
     // I originally had this as a DCHECK but I saw a failure once that I
     // can't reproduce. It looks like it might be related to the
     // navigation controller only keeping a limited number of navigation
     // events. For now we'll just attach a feature specifying that this is
     // a mismatch and try and figure out what to do with it on the server.
     DLOG(WARNING) << "Expected:" << entry->GetURL()
                  << " Actual:" << redirect_chain.back();
     AddFeature(feature_prefix + features::kRedirectUrlMismatch,
                1.0,
                request);
     return;
   }
   // We skip the last element since it should just be the current url.
   for (size_t i = 0; i < redirect_chain.size() - 1; i++) {
     std::string printable_redirect = redirect_chain[i].spec();
     if (redirect_chain[i].SchemeIsCryptographic()) {
       printable_redirect = features::kSecureRedirectValue;
     }
     AddFeature(base::StringPrintf("%s%s[%" PRIuS "]=%s",
                                   feature_prefix.c_str(),
                                   features::kRedirect,
                                   i,
                                   printable_redirect.c_str()),
                1.0,
                request);
   }
 }

 BrowserFeatureExtractor::BrowserFeatureExtractor(
     WebContents* tab,
     ClientSideDetectionHost* host)
     : tab_(tab),
       host_(host),
       weak_factory_(this) {
   DCHECK(tab);
 }

 BrowserFeatureExtractor::~BrowserFeatureExtractor() {
   weak_factory_.InvalidateWeakPtrs();
 }

 void BrowserFeatureExtractor::ExtractFeatures(const BrowseInfo* info,
                                               ClientPhishingRequest* request,
                                               const DoneCallback& callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   DCHECK(request);
   DCHECK(info);
   DCHECK(GURL(request->url()).SchemeIsHTTPOrHTTPS());
   DCHECK(!callback.is_null());
   // Extract features pertaining to this navigation.
   const NavigationController& controller = tab_->GetController();
   int url_index = -1;
   int first_host_index = -1;

   GURL request_url(request->url());
   int index = controller.GetCurrentEntryIndex();
   // The url that we are extracting features for should already be commited.
   DCHECK_NE(index, -1);
   for (; index >= 0; index--) {
     NavigationEntry* entry = controller.GetEntryAtIndex(index);
     if (url_index == -1 && entry->GetURL() == request_url) {
       // It's possible that we've been on the on the possibly phishy url before
       // in this tab, so make sure that we use the latest navigation for
       // features.
       // Note that it's possible that the url_index should always be the
       // latest entry, but I'm worried about possible races during a navigation
       // and transient entries (i.e. interstiatials) so for now we will just
       // be cautious.
       url_index = index;
     } else if (index < url_index) {
       if (entry->GetURL().host_piece() == request_url.host_piece()) {
         first_host_index = index;
       } else {
         // We have found the possibly phishing url, but we are no longer on the
         // host. No reason to look back any further.
         break;
       }
     }
   }

   // Add features pertaining to how we got to
   //   1) The candidate url
   //   2) The first url on the same host as the candidate url (assuming that
   //      it's different from the candidate url).
   if (url_index != -1) {
     AddNavigationFeatures(
         std::string(), controller, url_index, info->url_redirects, request);
   }
   if (first_host_index != -1) {
     AddNavigationFeatures(features::kHostPrefix,
                           controller,
                           first_host_index,
                           info->host_redirects,
                           request);
   }

   // The API doesn't take a std::unique_ptr because the API gets mocked and we
   // cannot mock an API that takes std::unique_ptr as arguments.
   std::unique_ptr<ClientPhishingRequest> req(request);

   ExtractBrowseInfoFeatures(*info, request);
   base::ThreadTaskRunnerHandle::Get()->PostTask(
       FROM_HERE,
       base::BindOnce(&BrowserFeatureExtractor::StartExtractFeatures,
                      weak_factory_.GetWeakPtr(), std::move(req), callback));
 }

 void BrowserFeatureExtractor::ExtractMalwareFeatures(
     BrowseInfo* info,
     ClientMalwareRequest* request,
     const MalwareDoneCallback& callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   DCHECK(!callback.is_null());

   // Grab the IPs because they might go away before we're done
   // checking them against the IP blacklist on the IO thread.
   std::unique_ptr<IPUrlMap> ips(new IPUrlMap);
   ips->swap(info->ips);

   IPUrlMap* ips_ptr = ips.get();

   // The API doesn't take a std::unique_ptr because the API gets mocked and we
   // cannot mock an API that takes std::unique_ptr as arguments.
   std::unique_ptr<ClientMalwareRequest> req(request);

   // IP blacklist lookups have to happen on the IO thread.
   BrowserThread::PostTaskAndReply(
       BrowserThread::IO, FROM_HERE,
       base::BindOnce(&FilterBenignIpsOnIOThread, host_->database_manager(),
                      ips_ptr),
       base::BindOnce(&BrowserFeatureExtractor::FinishExtractMalwareFeatures,
                      weak_factory_.GetWeakPtr(), std::move(ips), callback,
                      std::move(req)));
 }

 void BrowserFeatureExtractor::ExtractBrowseInfoFeatures(
     const BrowseInfo& info,
     ClientPhishingRequest* request) {
   if (info.unsafe_resource.get()) {
     // A SafeBrowsing interstitial was shown for the current URL.
     AddFeature(features::kSafeBrowsingMaliciousUrl +
                info.unsafe_resource->url.spec(),
                1.0,
                request);
     AddFeature(features::kSafeBrowsingOriginalUrl +
                info.unsafe_resource->original_url.spec(),
                1.0,
                request);
     AddFeature(features::kSafeBrowsingIsSubresource,
                info.unsafe_resource->is_subresource ? 1.0 : 0.0,
                request);
     AddFeature(features::kSafeBrowsingThreatType,
                static_cast<double>(info.unsafe_resource->threat_type),
                request);
   }
   if (info.http_status_code != 0) {
     AddFeature(features::kHttpStatusCode, info.http_status_code, request);
   }
 }

 void BrowserFeatureExtractor::StartExtractFeatures(
     std::unique_ptr<ClientPhishingRequest> request,
     const DoneCallback& callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   history::HistoryService* history;
   if (!request || !request->IsInitialized() || !GetHistoryService(&history)) {
     callback.Run(false, std::move(request));
     return;
   }
   GURL request_url(request->url());
   history->QueryURL(request_url,
                     true /* wants_visits */,
                     base::Bind(&BrowserFeatureExtractor::QueryUrlHistoryDone,
                                base::Unretained(this),
                                base::Passed(&request),
                                callback),
                     &cancelable_task_tracker_);
 }

 void BrowserFeatureExtractor::QueryUrlHistoryDone(
     std::unique_ptr<ClientPhishingRequest> request,
     const DoneCallback& callback,
     bool success,
     const history::URLRow& row,
     const history::VisitVector& visits) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   DCHECK(request);
   DCHECK(!callback.is_null());
   if (!success) {
     // URL is not found in the history.  In practice this should not
     // happen (unless there is a real error) because we just visited
     // that URL.
     callback.Run(false, std::move(request));
     return;
   }
   AddFeature(features::kUrlHistoryVisitCount,
              static_cast<double>(row.visit_count()),
              request.get());

   base::Time threshold = base::Time::Now() - base::TimeDelta::FromDays(1);
   int num_visits_24h_ago = 0;
   int num_visits_typed = 0;
   int num_visits_link = 0;
   for (history::VisitVector::const_iterator it = visits.begin();
        it != visits.end();
        ++it) {
     if (!ui::PageTransitionIsMainFrame(it->transition)) {
       continue;
     }
     if (it->visit_time < threshold) {
       ++num_visits_24h_ago;
     }
     if (ui::PageTransitionCoreTypeIs(it->transition,
                                      ui::PAGE_TRANSITION_TYPED)) {
       ++num_visits_typed;
     } else if (ui::PageTransitionCoreTypeIs(it->transition,
                                             ui::PAGE_TRANSITION_LINK)) {
       ++num_visits_link;
     }
   }
   AddFeature(features::kUrlHistoryVisitCountMoreThan24hAgo,
              static_cast<double>(num_visits_24h_ago),
              request.get());
   AddFeature(features::kUrlHistoryTypedCount,
              static_cast<double>(num_visits_typed),
              request.get());
   AddFeature(features::kUrlHistoryLinkCount,
              static_cast<double>(num_visits_link),
              request.get());

   // Issue next history lookup for host visits.
   history::HistoryService* history;
   if (!GetHistoryService(&history)) {
     callback.Run(false, std::move(request));
     return;
   }
   GURL::Replacements rep;
   rep.SetSchemeStr(url::kHttpScheme);
   GURL http_url = GURL(request->url()).ReplaceComponents(rep);
   history->GetVisibleVisitCountToHost(
       http_url,
       base::Bind(&BrowserFeatureExtractor::QueryHttpHostVisitsDone,
                  base::Unretained(this), base::Passed(&request), callback),
       &cancelable_task_tracker_);
 }

 void BrowserFeatureExtractor::QueryHttpHostVisitsDone(
     std::unique_ptr<ClientPhishingRequest> request,
     const DoneCallback& callback,
     bool success,
     int num_visits,
     base::Time first_visit) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   DCHECK(request);
   DCHECK(!callback.is_null());
   if (!success) {
     callback.Run(false, std::move(request));
     return;
   }
   SetHostVisitsFeatures(num_visits, first_visit, true, request.get());

   // Same lookup but for the HTTPS URL.
   history::HistoryService* history;
   if (!GetHistoryService(&history)) {
     callback.Run(false, std::move(request));
     return;
   }
   GURL::Replacements rep;
   rep.SetSchemeStr(url::kHttpsScheme);
   GURL https_url = GURL(request->url()).ReplaceComponents(rep);
   history->GetVisibleVisitCountToHost(
       https_url,
       base::Bind(&BrowserFeatureExtractor::QueryHttpsHostVisitsDone,
                  base::Unretained(this), base::Passed(&request), callback),
       &cancelable_task_tracker_);
 }

 void BrowserFeatureExtractor::QueryHttpsHostVisitsDone(
     std::unique_ptr<ClientPhishingRequest> request,
     const DoneCallback& callback,
     bool success,
     int num_visits,
     base::Time first_visit) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   DCHECK(request);
   DCHECK(!callback.is_null());
   if (!success) {
     callback.Run(false, std::move(request));
     return;
   }
   SetHostVisitsFeatures(num_visits, first_visit, false, request.get());
   callback.Run(true, std::move(request));
 }

 void BrowserFeatureExtractor::SetHostVisitsFeatures(
     int num_visits,
     base::Time first_visit,
     bool is_http_query,
     ClientPhishingRequest* request) {
   DCHECK(request);
   AddFeature(is_http_query ?
              features::kHttpHostVisitCount : features::kHttpsHostVisitCount,
              static_cast<double>(num_visits),
              request);
   if (num_visits > 0) {
     AddFeature(
         is_http_query ?
         features::kFirstHttpHostVisitMoreThan24hAgo :
         features::kFirstHttpsHostVisitMoreThan24hAgo,
         (first_visit < (base::Time::Now() - base::TimeDelta::FromDays(1))) ?
         1.0 : 0.0,
         request);
   }
 }

 bool BrowserFeatureExtractor::GetHistoryService(
     history::HistoryService** history) {
   *history = NULL;
   if (tab_ && tab_->GetBrowserContext()) {
     Profile* profile = Profile::FromBrowserContext(tab_->GetBrowserContext());
     *history = HistoryServiceFactory::GetForProfile(
         profile, ServiceAccessType::EXPLICIT_ACCESS);
     if (*history) {
       return true;
     }
   }
   DVLOG(2) << "Unable to query history.  No history service available.";
   return false;
 }

 void BrowserFeatureExtractor::FinishExtractMalwareFeatures(
     std::unique_ptr<IPUrlMap> bad_ips,
     MalwareDoneCallback callback,
     std::unique_ptr<ClientMalwareRequest> request) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   int matched_bad_ips = 0;
   for (IPUrlMap::const_iterator it = bad_ips->begin();
        it != bad_ips->end(); ++it) {
     AddMalwareIpUrlInfo(it->first, it->second, request.get());
     ++matched_bad_ips;
     // Limit the number of matched bad IPs in one request to control
     // the request's size
     if (matched_bad_ips >= kMaxMalwareIPPerRequest) {
       break;
     }
   }
   callback.Run(true, std::move(request));
 }

 }  // namespace safe_browsing
	// Copyright (c) 2012 The Chromium Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	#include "chrome/browser/safe_browsing/browser_feature_extractor.h"

	#include <stddef.h>

	#include <map>
	#include <utility>

	#include "base/bind.h"
	#include "base/bind_helpers.h"
	#include "base/format_macros.h"
	#include "base/location.h"
	#include "base/single_thread_task_runner.h"
	#include "base/stl_util.h"
	#include "base/strings/stringprintf.h"
	#include "base/threading/thread_task_runner_handle.h"
	#include "base/time/time.h"
	#include "chrome/browser/history/history_service_factory.h"
	#include "chrome/browser/profiles/profile.h"
	#include "chrome/browser/safe_browsing/browser_features.h"
	#include "chrome/browser/safe_browsing/client_side_detection_host.h"
	#include "components/history/core/browser/history_service.h"
	#include "components/history/core/browser/history_types.h"
	#include "components/safe_browsing/db/database_manager.h"
	#include "components/safe_browsing/proto/csd.pb.h"
	#include "content/public/browser/browser_thread.h"
	#include "content/public/browser/navigation_controller.h"
	#include "content/public/browser/navigation_entry.h"
	#include "content/public/browser/web_contents.h"
	#include "ui/base/page_transition_types.h"
	#include "url/gurl.h"

	using content::BrowserThread;
	using content::NavigationController;
	using content::NavigationEntry;
	using content::ResourceType;
	using content::WebContents;

	namespace safe_browsing {

	namespace {

	const int kMaxMalwareIPPerRequest = 5;

	void FilterBenignIpsOnIOThread(
	scoped_refptr<SafeBrowsingDatabaseManager> database_manager,
	IPUrlMap* ips) {
	DCHECK_CURRENTLY_ON(BrowserThread::IO);
	for (IPUrlMap::iterator it = ips->begin(); it != ips->end();) {
	if (!database_manager.get() \|\|
	!database_manager->MatchMalwareIP(it->first)) {
	// it++ here returns a copy of the old iterator and passes it to erase.
	ips->erase(it++);
	} else {
	++it;
	}
	}
	}
	} // namespace

	IPUrlInfo::IPUrlInfo(const std::string& url,
	const std::string& method,
	const std::string& referrer,
	const ResourceType& resource_type)
	: url(url),
	method(method),
	referrer(referrer),
	resource_type(resource_type) {
	}

	IPUrlInfo::IPUrlInfo(const IPUrlInfo& other) = default;

	IPUrlInfo::~IPUrlInfo() {}

	BrowseInfo::BrowseInfo() : http_status_code(0) {}

	BrowseInfo::~BrowseInfo() {}

	static void AddFeature(const std::string& feature_name,
	double feature_value,
	ClientPhishingRequest* request) {
	DCHECK(request);
	ClientPhishingRequest::Feature* feature =
	request->add_non_model_feature_map();
	feature->set_name(feature_name);
	feature->set_value(feature_value);
	DVLOG(2) << "Browser feature: " << feature->name() << " " << feature->value();
	}

	static void AddMalwareIpUrlInfo(const std::string& ip,
	const std::vector<IPUrlInfo>& meta_infos,
	ClientMalwareRequest* request) {
	DCHECK(request);
	for (std::vector<IPUrlInfo>::const_iterator it = meta_infos.begin();
	it != meta_infos.end(); ++it) {
	ClientMalwareRequest::UrlInfo* urlinfo =
	request->add_bad_ip_url_info();
	// We add the information about url on the bad ip.
	urlinfo->set_ip(ip);
	urlinfo->set_url(it->url);
	urlinfo->set_method(it->method);
	urlinfo->set_referrer(it->referrer);
	urlinfo->set_resource_type(static_cast<int>(it->resource_type));
	}
	DVLOG(2) << "Added url info for bad ip: " << ip;
	}

	static void AddNavigationFeatures(
	const std::string& feature_prefix,
	const NavigationController& controller,
	int index,
	const std::vector<GURL>& redirect_chain,
	ClientPhishingRequest* request) {
	NavigationEntry* entry = controller.GetEntryAtIndex(index);
	bool is_secure_referrer = entry->GetReferrer().url.SchemeIsCryptographic();
	if (!is_secure_referrer) {
	AddFeature(base::StringPrintf("%s%s=%s",
	feature_prefix.c_str(),
	features::kReferrer,
	entry->GetReferrer().url.spec().c_str()),
	1.0,
	request);
	}
	AddFeature(feature_prefix + features::kHasSSLReferrer,
	is_secure_referrer ? 1.0 : 0.0,
	request);
	AddFeature(feature_prefix + features::kPageTransitionType,
	static_cast<double>(
	ui::PageTransitionStripQualifier(
	entry->GetTransitionType())),
	request);
	AddFeature(feature_prefix + features::kIsFirstNavigation,
	index == 0 ? 1.0 : 0.0,
	request);
	// Redirect chain should always be at least of size one, as the rendered
	// url is the last element in the chain.
	if (redirect_chain.empty()) {
	NOTREACHED();
	return;
	}
	if (redirect_chain.back() != entry->GetURL()) {
	// I originally had this as a DCHECK but I saw a failure once that I
	// can't reproduce. It looks like it might be related to the
	// navigation controller only keeping a limited number of navigation
	// events. For now we'll just attach a feature specifying that this is
	// a mismatch and try and figure out what to do with it on the server.
	DLOG(WARNING) << "Expected:" << entry->GetURL()
	<< " Actual:" << redirect_chain.back();
	AddFeature(feature_prefix + features::kRedirectUrlMismatch,
	1.0,
	request);
	return;
	}
	// We skip the last element since it should just be the current url.
	for (size_t i = 0; i < redirect_chain.size() - 1; i++) {
	std::string printable_redirect = redirect_chain[i].spec();
	if (redirect_chain[i].SchemeIsCryptographic()) {
	printable_redirect = features::kSecureRedirectValue;
	}
	AddFeature(base::StringPrintf("%s%s[%" PRIuS "]=%s",
	feature_prefix.c_str(),
	features::kRedirect,
	i,
	printable_redirect.c_str()),
	1.0,
	request);
	}
	}

	BrowserFeatureExtractor::BrowserFeatureExtractor(
	WebContents* tab,
	ClientSideDetectionHost* host)
	: tab_(tab),
	host_(host),
	weak_factory_(this) {
	DCHECK(tab);
	}

	BrowserFeatureExtractor::~BrowserFeatureExtractor() {
	weak_factory_.InvalidateWeakPtrs();
	}

	void BrowserFeatureExtractor::ExtractFeatures(const BrowseInfo* info,
	ClientPhishingRequest* request,
	const DoneCallback& callback) {
	DCHECK_CURRENTLY_ON(BrowserThread::UI);
	DCHECK(request);
	DCHECK(info);
	DCHECK(GURL(request->url()).SchemeIsHTTPOrHTTPS());
	DCHECK(!callback.is_null());
	// Extract features pertaining to this navigation.
	const NavigationController& controller = tab_->GetController();
	int url_index = -1;
	int first_host_index = -1;

	GURL request_url(request->url());
	int index = controller.GetCurrentEntryIndex();
	// The url that we are extracting features for should already be commited.
	DCHECK_NE(index, -1);
	for (; index >= 0; index--) {
	NavigationEntry* entry = controller.GetEntryAtIndex(index);
	if (url_index == -1 && entry->GetURL() == request_url) {
	// It's possible that we've been on the on the possibly phishy url before
	// in this tab, so make sure that we use the latest navigation for
	// features.
	// Note that it's possible that the url_index should always be the
	// latest entry, but I'm worried about possible races during a navigation
	// and transient entries (i.e. interstiatials) so for now we will just
	// be cautious.
	url_index = index;
	} else if (index < url_index) {
	if (entry->GetURL().host_piece() == request_url.host_piece()) {
	first_host_index = index;
	} else {
	// We have found the possibly phishing url, but we are no longer on the
	// host. No reason to look back any further.
	break;
	}
	}
	}

	// Add features pertaining to how we got to
	// 1) The candidate url
	// 2) The first url on the same host as the candidate url (assuming that
	// it's different from the candidate url).
	if (url_index != -1) {
	AddNavigationFeatures(
	std::string(), controller, url_index, info->url_redirects, request);
	}
	if (first_host_index != -1) {
	AddNavigationFeatures(features::kHostPrefix,
	controller,
	first_host_index,
	info->host_redirects,
	request);
	}

	// The API doesn't take a std::unique_ptr because the API gets mocked and we
	// cannot mock an API that takes std::unique_ptr as arguments.
	std::unique_ptr<ClientPhishingRequest> req(request);

	ExtractBrowseInfoFeatures(*info, request);
	base::ThreadTaskRunnerHandle::Get()->PostTask(
	FROM_HERE,
	base::BindOnce(&BrowserFeatureExtractor::StartExtractFeatures,
	weak_factory_.GetWeakPtr(), std::move(req), callback));
	}

	void BrowserFeatureExtractor::ExtractMalwareFeatures(
	BrowseInfo* info,
	ClientMalwareRequest* request,
	const MalwareDoneCallback& callback) {
	DCHECK_CURRENTLY_ON(BrowserThread::UI);
	DCHECK(!callback.is_null());

	// Grab the IPs because they might go away before we're done
	// checking them against the IP blacklist on the IO thread.
	std::unique_ptr<IPUrlMap> ips(new IPUrlMap);
	ips->swap(info->ips);

	IPUrlMap* ips_ptr = ips.get();

	// The API doesn't take a std::unique_ptr because the API gets mocked and we
	// cannot mock an API that takes std::unique_ptr as arguments.
	std::unique_ptr<ClientMalwareRequest> req(request);

	// IP blacklist lookups have to happen on the IO thread.
	BrowserThread::PostTaskAndReply(
	BrowserThread::IO, FROM_HERE,
	base::BindOnce(&FilterBenignIpsOnIOThread, host_->database_manager(),
	ips_ptr),
	base::BindOnce(&BrowserFeatureExtractor::FinishExtractMalwareFeatures,
	weak_factory_.GetWeakPtr(), std::move(ips), callback,
	std::move(req)));
	}

	void BrowserFeatureExtractor::ExtractBrowseInfoFeatures(
	const BrowseInfo& info,
	ClientPhishingRequest* request) {
	if (info.unsafe_resource.get()) {
	// A SafeBrowsing interstitial was shown for the current URL.
	AddFeature(features::kSafeBrowsingMaliciousUrl +
	info.unsafe_resource->url.spec(),
	1.0,
	request);
	AddFeature(features::kSafeBrowsingOriginalUrl +
	info.unsafe_resource->original_url.spec(),
	1.0,
	request);
	AddFeature(features::kSafeBrowsingIsSubresource,
	info.unsafe_resource->is_subresource ? 1.0 : 0.0,
	request);
	AddFeature(features::kSafeBrowsingThreatType,
	static_cast<double>(info.unsafe_resource->threat_type),
	request);
	}
	if (info.http_status_code != 0) {
	AddFeature(features::kHttpStatusCode, info.http_status_code, request);
	}
	}

	void BrowserFeatureExtractor::StartExtractFeatures(
	std::unique_ptr<ClientPhishingRequest> request,
	const DoneCallback& callback) {
	DCHECK_CURRENTLY_ON(BrowserThread::UI);
	history::HistoryService* history;
	if (!request \|\| !request->IsInitialized() \|\| !GetHistoryService(&history)) {
	callback.Run(false, std::move(request));
	return;
	}
	GURL request_url(request->url());
	history->QueryURL(request_url,
	true /* wants_visits */,
	base::Bind(&BrowserFeatureExtractor::QueryUrlHistoryDone,
	base::Unretained(this),
	base::Passed(&request),
	callback),
	&cancelable_task_tracker_);
	}

	void BrowserFeatureExtractor::QueryUrlHistoryDone(
	std::unique_ptr<ClientPhishingRequest> request,
	const DoneCallback& callback,
	bool success,
	const history::URLRow& row,
	const history::VisitVector& visits) {
	DCHECK_CURRENTLY_ON(BrowserThread::UI);
	DCHECK(request);
	DCHECK(!callback.is_null());
	if (!success) {
	// URL is not found in the history. In practice this should not
	// happen (unless there is a real error) because we just visited
	// that URL.
	callback.Run(false, std::move(request));
	return;
	}
	AddFeature(features::kUrlHistoryVisitCount,
	static_cast<double>(row.visit_count()),
	request.get());

	base::Time threshold = base::Time::Now() - base::TimeDelta::FromDays(1);
	int num_visits_24h_ago = 0;
	int num_visits_typed = 0;
	int num_visits_link = 0;
	for (history::VisitVector::const_iterator it = visits.begin();
	it != visits.end();
	++it) {
	if (!ui::PageTransitionIsMainFrame(it->transition)) {
	continue;
	}
	if (it->visit_time < threshold) {
	++num_visits_24h_ago;
	}
	if (ui::PageTransitionCoreTypeIs(it->transition,
	ui::PAGE_TRANSITION_TYPED)) {
	++num_visits_typed;
	} else if (ui::PageTransitionCoreTypeIs(it->transition,
	ui::PAGE_TRANSITION_LINK)) {
	++num_visits_link;
	}
	}
	AddFeature(features::kUrlHistoryVisitCountMoreThan24hAgo,
	static_cast<double>(num_visits_24h_ago),
	request.get());
	AddFeature(features::kUrlHistoryTypedCount,
	static_cast<double>(num_visits_typed),
	request.get());
	AddFeature(features::kUrlHistoryLinkCount,
	static_cast<double>(num_visits_link),
	request.get());

	// Issue next history lookup for host visits.
	history::HistoryService* history;
	if (!GetHistoryService(&history)) {
	callback.Run(false, std::move(request));
	return;
	}
	GURL::Replacements rep;
	rep.SetSchemeStr(url::kHttpScheme);
	GURL http_url = GURL(request->url()).ReplaceComponents(rep);
	history->GetVisibleVisitCountToHost(
	http_url,
	base::Bind(&BrowserFeatureExtractor::QueryHttpHostVisitsDone,
	base::Unretained(this), base::Passed(&request), callback),
	&cancelable_task_tracker_);
	}

	void BrowserFeatureExtractor::QueryHttpHostVisitsDone(
	std::unique_ptr<ClientPhishingRequest> request,
	const DoneCallback& callback,
	bool success,
	int num_visits,
	base::Time first_visit) {
	DCHECK_CURRENTLY_ON(BrowserThread::UI);
	DCHECK(request);
	DCHECK(!callback.is_null());
	if (!success) {
	callback.Run(false, std::move(request));
	return;
	}
	SetHostVisitsFeatures(num_visits, first_visit, true, request.get());

	// Same lookup but for the HTTPS URL.
	history::HistoryService* history;
	if (!GetHistoryService(&history)) {
	callback.Run(false, std::move(request));
	return;
	}
	GURL::Replacements rep;
	rep.SetSchemeStr(url::kHttpsScheme);
	GURL https_url = GURL(request->url()).ReplaceComponents(rep);
	history->GetVisibleVisitCountToHost(
	https_url,
	base::Bind(&BrowserFeatureExtractor::QueryHttpsHostVisitsDone,
	base::Unretained(this), base::Passed(&request), callback),
	&cancelable_task_tracker_);
	}

	void BrowserFeatureExtractor::QueryHttpsHostVisitsDone(
	std::unique_ptr<ClientPhishingRequest> request,
	const DoneCallback& callback,
	bool success,
	int num_visits,
	base::Time first_visit) {
	DCHECK_CURRENTLY_ON(BrowserThread::UI);
	DCHECK(request);
	DCHECK(!callback.is_null());
	if (!success) {
	callback.Run(false, std::move(request));
	return;
	}
	SetHostVisitsFeatures(num_visits, first_visit, false, request.get());
	callback.Run(true, std::move(request));
	}

	void BrowserFeatureExtractor::SetHostVisitsFeatures(
	int num_visits,
	base::Time first_visit,
	bool is_http_query,
	ClientPhishingRequest* request) {
	DCHECK(request);
	AddFeature(is_http_query ?
	features::kHttpHostVisitCount : features::kHttpsHostVisitCount,
	static_cast<double>(num_visits),
	request);
	if (num_visits > 0) {
	AddFeature(
	is_http_query ?
	features::kFirstHttpHostVisitMoreThan24hAgo :
	features::kFirstHttpsHostVisitMoreThan24hAgo,
	(first_visit < (base::Time::Now() - base::TimeDelta::FromDays(1))) ?
	1.0 : 0.0,
	request);
	}
	}

	bool BrowserFeatureExtractor::GetHistoryService(
	history::HistoryService** history) {
	*history = NULL;
	if (tab_ && tab_->GetBrowserContext()) {
	Profile* profile = Profile::FromBrowserContext(tab_->GetBrowserContext());
	*history = HistoryServiceFactory::GetForProfile(
	profile, ServiceAccessType::EXPLICIT_ACCESS);
	if (*history) {
	return true;
	}
	}
	DVLOG(2) << "Unable to query history. No history service available.";
	return false;
	}

	void BrowserFeatureExtractor::FinishExtractMalwareFeatures(
	std::unique_ptr<IPUrlMap> bad_ips,
	MalwareDoneCallback callback,
	std::unique_ptr<ClientMalwareRequest> request) {
	DCHECK_CURRENTLY_ON(BrowserThread::UI);
	int matched_bad_ips = 0;
	for (IPUrlMap::const_iterator it = bad_ips->begin();
	it != bad_ips->end(); ++it) {
	AddMalwareIpUrlInfo(it->first, it->second, request.get());
	++matched_bad_ips;
	// Limit the number of matched bad IPs in one request to control
	// the request's size
	if (matched_bad_ips >= kMaxMalwareIPPerRequest) {
	break;
	}
	}
	callback.Run(true, std::move(request));
	}

	} // namespace safe_browsing