[NTP] Fix article suggestion clicks contributing to Most Visited tiles

There's a need to distinguish clicks on different elements on the NTP:
a) clicks on Most Visited tiles.
b) clicks on (newly introduced) article suggestions (aka snippets).

The first should contribute to Most Visited tiles (i.e. boost tiles
that have been clicked in the past). The second shouldn't.

We choose to achieve this by specifying a referrer for article
suggestion clicks. This exposes the referrer to third parties, which
has been discussed and considered a desirable feature.

The fix relies on such a workaround due to the current lack of
infrastructure to propagate opaque feature-specific data from upper
layers to navigation history (and sync).

The approach competes with more intrusive/controversial alternatives to
achieve the same:

1. Use page transition types (LINK vs AUTO_BOOKMARK) to distinguish
   tile clicks from article suggestion clicks: unfortunately both types
   have been used in the past (older versions of Chrome).

2. Introducing a new page transition type or qualifier: this can be
   considered a layering violation.

3. Introduce a dummy redirect by means of a data: schema page.

BUG=645895

Review-Url: https://codereview.chromium.org/2338133006
Cr-Commit-Position: refs/heads/master@{#419242}
diff --git a/chrome/android/java/src/org/chromium/chrome/browser/ntp/NewTabPage.java b/chrome/android/java/src/org/chromium/chrome/browser/ntp/NewTabPage.java
index ff1a75b..f5b0d27 100644
--- a/chrome/android/java/src/org/chromium/chrome/browser/ntp/NewTabPage.java
+++ b/chrome/android/java/src/org/chromium/chrome/browser/ntp/NewTabPage.java
@@ -45,6 +45,7 @@
 import org.chromium.chrome.browser.ntp.LogoBridge.Logo;
 import org.chromium.chrome.browser.ntp.LogoBridge.LogoObserver;
 import org.chromium.chrome.browser.ntp.NewTabPageView.NewTabPageManager;
+import org.chromium.chrome.browser.ntp.snippets.KnownCategories;
 import org.chromium.chrome.browser.ntp.snippets.SnippetArticle;
 import org.chromium.chrome.browser.ntp.snippets.SnippetsBridge;
 import org.chromium.chrome.browser.ntp.snippets.SnippetsConfig;
@@ -68,6 +69,7 @@
 import org.chromium.chrome.browser.tabmodel.document.TabDelegate;
 import org.chromium.chrome.browser.util.UrlUtilities;
 import org.chromium.content_public.browser.LoadUrlParams;
+import org.chromium.content_public.common.Referrer;
 import org.chromium.net.NetworkChangeNotifier;
 import org.chromium.ui.WindowOpenDisposition;
 import org.chromium.ui.base.DeviceFormFactor;
@@ -101,6 +103,9 @@
     private static final int CTA_IMAGE_CLICKED = 1;
     private static final int ANIMATED_LOGO_CLICKED = 2;
 
+    private static final String CHROME_CONTENT_SUGGESTIONS_REFERRER =
+            "https://www.googleapis.com/auth/chrome-content-suggestions";
+
     private static MostVisitedSites sMostVisitedSitesForTests;
 
     private final Tab mTab;
@@ -239,7 +244,7 @@
             recordOpenedMostVisitedItem(item);
             String url = item.getUrl();
             if (!switchToExistingTab(url)) {
-                openUrl(WindowOpenDisposition.CURRENT_TAB, url);
+                openUrlMostVisited(WindowOpenDisposition.CURRENT_TAB, url);
             }
         }
 
@@ -248,7 +253,9 @@
             if (mIsDestroyed) return;
             NewTabPageUma.recordAction(NewTabPageUma.ACTION_CLICKED_LEARN_MORE);
             String url = "https://support.google.com/chrome/?p=new_tab";
-            openUrl(WindowOpenDisposition.CURRENT_TAB, url);
+            // TODO(mastiz): Change this to LINK?
+            openUrl(WindowOpenDisposition.CURRENT_TAB,
+                    new LoadUrlParams(url, PageTransition.AUTO_BOOKMARK));
         }
 
         @TargetApi(Build.VERSION_CODES.LOLLIPOP)
@@ -312,26 +319,42 @@
                     article.mPosition, article.mPublishTimestampMilliseconds, article.mScore,
                     windowOpenDisposition);
             NewTabPageUma.monitorContentSuggestionVisit(mTab, article.mCategory);
-            openUrl(windowOpenDisposition, article.mUrl);
+            LoadUrlParams loadUrlParams =
+                    new LoadUrlParams(article.mUrl, PageTransition.AUTO_BOOKMARK);
+
+            // For article suggestions, we set the referrer. This is exploited
+            // to filter out these history entries for NTP tiles.
+            // TODO(mastiz): Extend this with support for other categories.
+            if (article.mCategory == KnownCategories.ARTICLES) {
+                loadUrlParams.setReferrer(new Referrer(
+                        CHROME_CONTENT_SUGGESTIONS_REFERRER, Referrer.REFERRER_POLICY_ALWAYS));
+            }
+
+            openUrl(windowOpenDisposition, loadUrlParams);
         }
 
-        private void openUrl(int windowOpenDisposition, String url) {
+        // TODO(mastiz): Merge with openMostVisitedItem().
+        private void openUrlMostVisited(int windowOpenDisposition, String url) {
+            openUrl(windowOpenDisposition, new LoadUrlParams(url, PageTransition.AUTO_BOOKMARK));
+        }
+
+        private void openUrl(int windowOpenDisposition, LoadUrlParams loadUrlParams) {
             assert !mIsDestroyed;
             switch (windowOpenDisposition) {
                 case WindowOpenDisposition.CURRENT_TAB:
-                    mTab.loadUrl(new LoadUrlParams(url, PageTransition.AUTO_BOOKMARK));
+                    mTab.loadUrl(loadUrlParams);
                     break;
                 case WindowOpenDisposition.NEW_FOREGROUND_TAB:
-                    openUrlInNewTab(url, false);
+                    openUrlInNewTab(loadUrlParams, false);
                     break;
                 case WindowOpenDisposition.OFF_THE_RECORD:
-                    openUrlInNewTab(url, true);
+                    openUrlInNewTab(loadUrlParams, true);
                     break;
                 case WindowOpenDisposition.NEW_WINDOW:
-                    openUrlInNewWindow(url);
+                    openUrlInNewWindow(loadUrlParams);
                     break;
                 case WindowOpenDisposition.SAVE_TO_DISK:
-                    saveUrlForOffline(url);
+                    saveUrlForOffline(loadUrlParams.getUrl());
                     break;
                 default:
                     assert false;
@@ -363,15 +386,15 @@
             switch (menuId) {
                 case ID_OPEN_IN_NEW_WINDOW:
                     // TODO(treib): Should we call recordOpenedMostVisitedItem here?
-                    openUrl(WindowOpenDisposition.NEW_WINDOW, item.getUrl());
+                    openUrlMostVisited(WindowOpenDisposition.NEW_WINDOW, item.getUrl());
                     return true;
                 case ID_OPEN_IN_NEW_TAB:
                     recordOpenedMostVisitedItem(item);
-                    openUrl(WindowOpenDisposition.NEW_FOREGROUND_TAB, item.getUrl());
+                    openUrlMostVisited(WindowOpenDisposition.NEW_FOREGROUND_TAB, item.getUrl());
                     return true;
                 case ID_OPEN_IN_INCOGNITO_TAB:
                     recordOpenedMostVisitedItem(item);
-                    openUrl(WindowOpenDisposition.OFF_THE_RECORD, item.getUrl());
+                    openUrlMostVisited(WindowOpenDisposition.OFF_THE_RECORD, item.getUrl());
                     return true;
                 case ID_REMOVE:
                     mMostVisitedSites.addBlacklistedUrl(item.getUrl());
@@ -392,16 +415,14 @@
             return PrefServiceBridge.getInstance().isIncognitoModeEnabled();
         }
 
-        private void openUrlInNewWindow(String url) {
+        private void openUrlInNewWindow(LoadUrlParams loadUrlParams) {
             TabDelegate tabDelegate = new TabDelegate(false);
-            // TODO(treib): Should this use PageTransition.AUTO_BOOKMARK?
-            LoadUrlParams loadUrlParams = new LoadUrlParams(url);
             tabDelegate.createTabInOtherWindow(loadUrlParams, mActivity, mTab.getParentId());
         }
 
-        private void openUrlInNewTab(String url, boolean incognito) {
-            mTabModelSelector.openNewTab(new LoadUrlParams(url, PageTransition.AUTO_BOOKMARK),
-                    TabLaunchType.FROM_LONGPRESS_BACKGROUND, mTab, incognito);
+        private void openUrlInNewTab(LoadUrlParams loadUrlParams, boolean incognito) {
+            mTabModelSelector.openNewTab(
+                    loadUrlParams, TabLaunchType.FROM_LONGPRESS_BACKGROUND, mTab, incognito);
         }
 
         private void saveUrlForOffline(String url) {
diff --git a/chrome/browser/history/history_tab_helper.cc b/chrome/browser/history/history_tab_helper.cc
index c633e3c..9e19218 100644
--- a/chrome/browser/history/history_tab_helper.cc
+++ b/chrome/browser/history/history_tab_helper.cc
@@ -30,6 +30,14 @@
 
 DEFINE_WEB_CONTENTS_USER_DATA_KEY(HistoryTabHelper);
 
+namespace {
+
+// Referrer used for clicks on article suggestions on the NTP.
+const char kChromeContentSuggestionsReferrer[] =
+    "https://www.googleapis.com/auth/chrome-content-suggestions";
+
+}  // namespace
+
 HistoryTabHelper::HistoryTabHelper(WebContents* web_contents)
     : content::WebContentsObserver(web_contents),
       received_page_title_(false) {
@@ -58,10 +66,16 @@
     bool did_replace_entry,
     int nav_entry_id,
     const content::FrameNavigateParams& params) {
+  // Clicks on content suggestions on the NTP should not contribute to the
+  // Most Visited tiles in the NTP.
+  const bool consider_for_ntp_most_visited =
+      params.referrer.url != GURL(kChromeContentSuggestionsReferrer);
+
   history::HistoryAddPageArgs add_page_args(
       params.url, timestamp, history::ContextIDForWebContents(web_contents()),
       nav_entry_id, params.referrer.url, params.redirects, params.transition,
-      history::SOURCE_BROWSED, did_replace_entry);
+      history::SOURCE_BROWSED, did_replace_entry,
+      consider_for_ntp_most_visited);
   if (ui::PageTransitionIsMainFrame(params.transition) &&
       virtual_url != params.url) {
     // Hack on the "virtual" URL so that it will appear in history. For some
diff --git a/chrome/browser/supervised_user/supervised_user_navigation_observer.cc b/chrome/browser/supervised_user/supervised_user_navigation_observer.cc
index e354bbb..8211965 100644
--- a/chrome/browser/supervised_user/supervised_user_navigation_observer.cc
+++ b/chrome/browser/supervised_user/supervised_user_navigation_observer.cc
@@ -68,7 +68,7 @@
   history::HistoryAddPageArgs add_page_args(
       url, timestamp, history::ContextIDForWebContents(web_contents_), 0, url,
       history::RedirectList(), ui::PAGE_TRANSITION_BLOCKED,
-      history::SOURCE_BROWSED, false);
+      history::SOURCE_BROWSED, false, true);
 
   // Add the entry to the history database.
   Profile* profile =
diff --git a/chrome/browser/ui/webui/web_dialog_web_contents_delegate_unittest.cc b/chrome/browser/ui/webui/web_dialog_web_contents_delegate_unittest.cc
index e419d3a..df6506a 100644
--- a/chrome/browser/ui/webui/web_dialog_web_contents_delegate_unittest.cc
+++ b/chrome/browser/ui/webui/web_dialog_web_contents_delegate_unittest.cc
@@ -66,7 +66,7 @@
   EXPECT_TRUE(test_web_contents_delegate_->IsPopupOrPanel(NULL));
   history::HistoryAddPageArgs should_add_args(
       GURL(), base::Time::Now(), 0, 0, GURL(), history::RedirectList(),
-      ui::PAGE_TRANSITION_TYPED, history::SOURCE_SYNCED, false);
+      ui::PAGE_TRANSITION_TYPED, history::SOURCE_SYNCED, false, true);
   test_web_contents_delegate_->NavigationStateChanged(
       NULL, content::InvalidateTypes(0));
   test_web_contents_delegate_->ActivateContents(NULL);