// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "components/ntp_snippets/contextual/contextual_suggestions_fetcher_impl.h"

#include <utility>
#include <vector>

#include "base/base64.h"
#include "base/bind_helpers.h"
#include "base/run_loop.h"
#include "base/strings/string_number_conversions.h"
#include "base/test/bind_test_util.h"
#include "base/test/metrics/histogram_tester.h"
#include "base/test/scoped_task_environment.h"
#include "components/ntp_snippets/contextual/contextual_suggestions_result.h"
#include "components/ntp_snippets/contextual/contextual_suggestions_test_utils.h"
#include "components/ntp_snippets/contextual/proto/chrome_search_api_request_context.pb.h"
#include "components/ntp_snippets/contextual/proto/get_pivots_request.pb.h"
#include "components/ntp_snippets/contextual/proto/get_pivots_response.pb.h"
#include "net/http/http_status_code.h"
#include "services/network/public/cpp/url_loader_completion_status.h"
#include "services/network/public/cpp/weak_wrapper_shared_url_loader_factory.h"
#include "services/network/test/test_url_loader_factory.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"

namespace contextual_suggestions {

using contextual_suggestions::AutoPeekConditions;
using contextual_suggestions::ClusterBuilder;
using contextual_suggestions::ContextualSuggestionsEvent;
using contextual_suggestions::ExploreContext;
using contextual_suggestions::GetPivotsQuery;
using contextual_suggestions::GetPivotsRequest;
using contextual_suggestions::GetPivotsResponse;
using contextual_suggestions::ImageId;
using contextual_suggestions::MockClustersCallback;
using contextual_suggestions::MockMetricsCallback;
using contextual_suggestions::PeekConditions;
using contextual_suggestions::PivotCluster;
using contextual_suggestions::PivotClusteringParams;
using contextual_suggestions::PivotDocument;
using contextual_suggestions::PivotDocumentParams;
using contextual_suggestions::PivotItem;
using contextual_suggestions::Pivots;
using network::SharedURLLoaderFactory;
using network::TestURLLoaderFactory;
using testing::ElementsAre;

namespace {

Cluster DefaultCluster() {
  return ClusterBuilder("Articles")
      .AddSuggestion(SuggestionBuilder(GURL("http://www.foobar.com"))
                         .Title("Title")
                         .PublisherName("cnn.com")
                         .Snippet("Summary")
                         .ImageId("abcdef")
                         .Build())
      .Build();
}

// Returns a single cluster with a single suggestion with preset values.
std::vector<Cluster> DefaultClusters() {
  std::vector<Cluster> clusters;

  clusters.emplace_back(DefaultCluster());
  return clusters;
}

void PopulateDocument(PivotDocument* document,
                      const ContextualSuggestion& suggestion) {
  document->mutable_url()->set_raw_url(suggestion.url.spec());
  document->set_title(suggestion.title);
  document->set_summary(suggestion.snippet);
  document->set_site_name(suggestion.publisher_name);

  ImageId* image_id = document->mutable_image()->mutable_id();
  image_id->set_encrypted_docid(suggestion.image_id);
}

void PopulatePeekConditions(AutoPeekConditions* proto_conditions,
                            const PeekConditions& peek_conditions) {
  proto_conditions->set_confidence(peek_conditions.confidence);
  proto_conditions->set_page_scroll_percentage(
      peek_conditions.page_scroll_percentage);
  proto_conditions->set_minimum_seconds_on_page(
      peek_conditions.minimum_seconds_on_page);
  proto_conditions->set_maximum_number_of_peeks(
      peek_conditions.maximum_number_of_peeks);
}

// Populates a GetPivotsResponse proto using |peek_text| and |clusters| and
// returns that proto as a serialized string.
std::string SerializedResponseProto(
    const std::string& peek_text,
    std::vector<Cluster> clusters,
    PeekConditions peek_conditions = PeekConditions(),
    ServerExperimentInfos experiment_infos = ServerExperimentInfos()) {
  GetPivotsResponse response_proto;
  Pivots* pivots = response_proto.mutable_pivots();
  PopulatePeekConditions(pivots->mutable_auto_peek_conditions(),
                         peek_conditions);
  pivots->mutable_peek_text()->set_text(peek_text);

  for (const auto& cluster : clusters) {
    PivotItem* root_item = pivots->add_item();
    PivotCluster* pivot_cluster = root_item->mutable_cluster();
    pivot_cluster->mutable_label()->set_label(cluster.title);
    for (const ContextualSuggestion& suggestion : cluster.suggestions) {
      PopulateDocument(pivot_cluster->add_item()->mutable_document(),
                       suggestion);
    }
  }

  for (const auto& experiment_info : experiment_infos) {
    ExperimentInfo* experiment = pivots->add_experiment_info();
    experiment->set_experiment_group_name(experiment_info.name);
    experiment->set_experiment_arm_name(experiment_info.group);
  }

  // The fetch parsing logic expects the response to come as (length, bytes)
  // where length is varint32 encoded, but ignores the actual length read.
  // " " is a valid varint32(32) so this works for now.
  // TODO(pnoland): Use a CodedOutputStream to prepend the actual size so that
  // we're not relying on implementation details.
  return " " + response_proto.SerializeAsString();
}

// Populates a GetPivotsResponse proto with a single, flat list of suggestions
// from |single_cluster| and returns that proto as a serialized string.
std::string SerializedResponseProto(const std::string& peek_text,
                                    Cluster single_cluster) {
  GetPivotsResponse response_proto;
  Pivots* pivots = response_proto.mutable_pivots();
  pivots->mutable_peek_text()->set_text(peek_text);

  for (const ContextualSuggestion& suggestion : single_cluster.suggestions) {
    PopulateDocument(pivots->add_item()->mutable_document(), suggestion);
  }

  // See explanation above for why we prepend " ".
  return " " + response_proto.SerializeAsString();
}

}  // namespace

class TestUrlKeyedDataCollectionConsentHelper
    : public unified_consent::UrlKeyedDataCollectionConsentHelper {
 public:
  TestUrlKeyedDataCollectionConsentHelper() = default;
  ~TestUrlKeyedDataCollectionConsentHelper() override = default;

  bool IsEnabled() override { return is_enabled_; }
  void SetIsEnabled(bool enabled) { is_enabled_ = enabled; }

 private:
  bool is_enabled_ = false;
};

class ContextualSuggestionsFetcherTest : public testing::Test {
 public:
  ContextualSuggestionsFetcherTest() {
    shared_url_loader_factory_ =
        base::MakeRefCounted<network::WeakWrapperSharedURLLoaderFactory>(
            &test_factory_);
    auto consent_helper =
        std::make_unique<TestUrlKeyedDataCollectionConsentHelper>();
    consent_helper_ = consent_helper.get();
    fetcher_ = std::make_unique<ContextualSuggestionsFetcherImpl>(
        shared_url_loader_factory_, std::move(consent_helper), "en");
  }

  ~ContextualSuggestionsFetcherTest() override {}

  void SetUp() override { consent_helper()->SetIsEnabled(true); }

  void SetFakeResponse(const std::string& response_data,
                       net::HttpStatusCode response_code = net::HTTP_OK,
                       network::URLLoaderCompletionStatus status =
                           network::URLLoaderCompletionStatus()) {
    GURL fetch_url(ContextualSuggestionsFetch::GetFetchEndpoint());
    network::ResourceResponseHead head;
    if (response_code >= 0) {
      head.headers = base::MakeRefCounted<net::HttpResponseHeaders>(
          "HTTP/1.1 " + base::NumberToString(response_code));
      status.decoded_body_length = response_data.length();
    }

    test_factory_.AddResponse(fetch_url, head, response_data, status);
  }

  void SendAndAwaitResponse(
      const GURL& context_url,
      MockClustersCallback* callback,
      MockMetricsCallback* mock_metrics_callback = nullptr) {
    ReportFetchMetricsCallback metrics_callback =
        mock_metrics_callback
            ? base::BindRepeating(&MockMetricsCallback::Report,
                                  base::Unretained(mock_metrics_callback))
            : base::DoNothing();
    fetcher().FetchContextualSuggestionsClusters(
        context_url, callback->ToOnceCallback(), metrics_callback);
    base::RunLoop().RunUntilIdle();
  }

  ContextualSuggestionsFetcher& fetcher() { return *fetcher_; }

  TestURLLoaderFactory* test_factory() { return &test_factory_; }

  TestUrlKeyedDataCollectionConsentHelper* consent_helper() {
    return consent_helper_;
  }

 private:
  base::test::ScopedTaskEnvironment scoped_task_environment_;
  network::TestURLLoaderFactory test_factory_;
  scoped_refptr<network::SharedURLLoaderFactory> shared_url_loader_factory_;
  TestUrlKeyedDataCollectionConsentHelper* consent_helper_;
  std::unique_ptr<ContextualSuggestionsFetcherImpl> fetcher_;

  DISALLOW_COPY_AND_ASSIGN(ContextualSuggestionsFetcherTest);
};

TEST_F(ContextualSuggestionsFetcherTest, SingleSuggestionResponse) {
  MockClustersCallback callback;
  MockMetricsCallback metrics_callback;
  SetFakeResponse(SerializedResponseProto("Peek Text", DefaultClusters()));

  SendAndAwaitResponse(GURL("http://www.article.com"), &callback,
                       &metrics_callback);

  EXPECT_TRUE(callback.has_run);
  ExpectResponsesMatch(
      std::move(callback),
      ContextualSuggestionsResult("Peek Text", DefaultClusters(),
                                  PeekConditions(), ServerExperimentInfos()));
  EXPECT_EQ(metrics_callback.events,
            std::vector<ContextualSuggestionsEvent>(
                {contextual_suggestions::FETCH_COMPLETED}));
}

TEST_F(ContextualSuggestionsFetcherTest,
       MultipleSuggestionMultipleClusterResponse) {
  std::vector<Cluster> clusters;
  std::vector<Cluster> clusters_copy;
  base::HistogramTester histogram_tester;

  ClusterBuilder cluster1_builder("Category 1");
  cluster1_builder
      .AddSuggestion(SuggestionBuilder(GURL("http://www.test.com"))
                         .Title("Title1")
                         .PublisherName("test.com")
                         .Snippet("Summary 1")
                         .ImageId("abc")
                         .Build())
      .AddSuggestion(SuggestionBuilder(GURL("http://www.foobar.com"))
                         .Title("Title2")
                         .PublisherName("foobar.com")
                         .Snippet("Summary 2")
                         .ImageId("def")
                         .Build());

  ClusterBuilder cluster2_builder("Category 2");
  cluster2_builder
      .AddSuggestion(SuggestionBuilder(GURL("http://www.barbaz.com"))
                         .Title("Title3")
                         .PublisherName("barbaz.com")
                         .Snippet("Summary 3")
                         .ImageId("ghi")
                         .Build())
      .AddSuggestion(SuggestionBuilder(GURL("http://www.cnn.com"))
                         .Title("Title4")
                         .PublisherName("cnn.com")
                         .Snippet("Summary 4")
                         .ImageId("jkl")
                         .Build())
      .AddSuggestion(SuggestionBuilder(GURL("http://www.slate.com"))
                         .Title("Title5")
                         .PublisherName("slate.com")
                         .Snippet("Summary 5")
                         .ImageId("mno")
                         .Build());
  ClusterBuilder c1_copy = cluster1_builder;
  ClusterBuilder c2_copy = cluster2_builder;

  clusters.emplace_back(cluster1_builder.Build());
  clusters.emplace_back(cluster2_builder.Build());

  clusters_copy.emplace_back(c1_copy.Build());
  clusters_copy.emplace_back(c2_copy.Build());

  SetFakeResponse(SerializedResponseProto("Peek Text", std::move(clusters)));
  MockClustersCallback callback;
  SendAndAwaitResponse(GURL("http://www.article.com/"), &callback);

  EXPECT_TRUE(callback.has_run);
  ExpectResponsesMatch(
      std::move(callback),
      ContextualSuggestionsResult("Peek Text", std::move(clusters_copy),
                                  PeekConditions(), ServerExperimentInfos()));

  histogram_tester.ExpectTotalCount(
      "ContextualSuggestions.FetchResponseNetworkBytes", 1);
  histogram_tester.ExpectTotalCount(
      "ContextualSuggestions.FetchLatencyMilliseconds", 1);
}

TEST_F(ContextualSuggestionsFetcherTest, FlatResponse) {
  SetFakeResponse(SerializedResponseProto("Peek Text", DefaultCluster()));
  MockClustersCallback callback;
  SendAndAwaitResponse(GURL("http://www.article.com/"), &callback);

  EXPECT_TRUE(callback.has_run);
  // There's no title for the flat/unclustered response case, since there's no
  // PivotCluster to copy it from. So we clear the expected title.
  std::vector<Cluster> expected_clusters = DefaultClusters();
  expected_clusters[0].title = "";
  ExpectResponsesMatch(
      std::move(callback),
      ContextualSuggestionsResult("Peek Text", std::move(expected_clusters),
                                  PeekConditions(), ServerExperimentInfos()));
}

TEST_F(ContextualSuggestionsFetcherTest, PeekConditionsAreParsed) {
  MockClustersCallback callback;
  MockMetricsCallback metrics_callback;
  PeekConditions peek_conditions;
  peek_conditions.confidence = 0.7;
  peek_conditions.page_scroll_percentage = 35.0;
  peek_conditions.minimum_seconds_on_page = 4.5;
  peek_conditions.maximum_number_of_peeks = 5.0;

  SetFakeResponse(
      SerializedResponseProto("Peek Text", DefaultClusters(), peek_conditions));

  SendAndAwaitResponse(GURL("http://www.article.com"), &callback,
                       &metrics_callback);

  EXPECT_TRUE(callback.has_run);
  ExpectResponsesMatch(
      std::move(callback),
      ContextualSuggestionsResult("Peek Text", DefaultClusters(),
                                  peek_conditions, ServerExperimentInfos()));
}

TEST_F(ContextualSuggestionsFetcherTest, ServerExperimentInfosAreParsed) {
  MockClustersCallback callback;
  MockMetricsCallback metrics_callback;
  ServerExperimentInfos experiment_infos;
  experiment_infos.emplace_back("trial1", "group1");
  experiment_infos.emplace_back("trial2", "group2");
  SetFakeResponse(SerializedResponseProto("Peek Text", DefaultClusters(),
                                          PeekConditions(), experiment_infos));

  SendAndAwaitResponse(GURL("http://www.article.com"), &callback,
                       &metrics_callback);

  EXPECT_TRUE(callback.has_run);
  ExpectResponsesMatch(
      std::move(callback),
      ContextualSuggestionsResult("Peek Text", DefaultClusters(),
                                  PeekConditions(), experiment_infos));
  EXPECT_EQ(metrics_callback.events,
            std::vector<ContextualSuggestionsEvent>(
                {contextual_suggestions::FETCH_COMPLETED}));
}

TEST_F(ContextualSuggestionsFetcherTest, HtmlEntitiesAreUnescaped) {
  ClusterBuilder cluster_builder("Category 1");
  cluster_builder.AddSuggestion(SuggestionBuilder(GURL("http://www.test.com"))
                                    .Title("&quot;foobar&quot;")
                                    .PublisherName("test.com")
                                    .Snippet("&#39;barbaz&#39;")
                                    .ImageId("abc")
                                    .Build());
  ClusterBuilder builder_copy = cluster_builder;
  SetFakeResponse(
      SerializedResponseProto("Peek Text", cluster_builder.Build()));
  MockClustersCallback callback;
  SendAndAwaitResponse(GURL("http://www.article.com/"), &callback);

  EXPECT_TRUE(callback.has_run);
  std::vector<Cluster> expected_clusters;
  expected_clusters.emplace_back(builder_copy.Build());
  // Clear the title since it's a flat response.
  expected_clusters[0].title = "";
  // Adjust the expected title and snippet to manually unescape the html
  // entities we added.
  expected_clusters[0].suggestions[0].title = "\"foobar\"";
  expected_clusters[0].suggestions[0].snippet = "\'barbaz\'";
  ExpectResponsesMatch(
      std::move(callback),
      ContextualSuggestionsResult("Peek Text", std::move(expected_clusters),
                                  PeekConditions(), ServerExperimentInfos()));
}

TEST_F(ContextualSuggestionsFetcherTest, RequestHeaderSetCorrectly) {
  net::HttpRequestHeaders headers;
  base::RunLoop interceptor_run_loop;
  base::HistogramTester histogram_tester;

  test_factory()->SetInterceptor(
      base::BindLambdaForTesting([&](const network::ResourceRequest& request) {
        headers = request.headers;
        interceptor_run_loop.Quit();
      }));
  SetFakeResponse(SerializedResponseProto("Peek Text", DefaultClusters()));

  MockClustersCallback callback;
  SendAndAwaitResponse(GURL("http://www.article.com/"), &callback);

  interceptor_run_loop.Run();

  std::string protobuf_header;
  ASSERT_TRUE(
      headers.GetHeader("X-Protobuffer-Request-Payload", &protobuf_header));

  std::string decoded_header_value;
  base::Base64Decode(protobuf_header, &decoded_header_value);
  GetPivotsRequest request;
  ASSERT_TRUE(request.ParseFromString(decoded_header_value));

  EXPECT_EQ(request.context().localization_context().language_code(), "en");
  EXPECT_EQ(request.query().context()[0].url(), "http://www.article.com/");
  EXPECT_TRUE(request.query().document_params().enabled());
  EXPECT_EQ(request.query().document_params().num(), 10);
  EXPECT_TRUE(request.query().document_params().enable_images());
  EXPECT_TRUE(request.query().clustering_params().enabled());
  EXPECT_TRUE(request.query().peek_text_params().enabled());

  histogram_tester.ExpectTotalCount(
      "ContextualSuggestions.FetchRequestProtoSizeKB", 1);
}

TEST_F(ContextualSuggestionsFetcherTest, CookiesIncludedWhenConsentIsEnabled) {
  network::ResourceRequest last_resource_request;

  test_factory()->SetInterceptor(
      base::BindLambdaForTesting([&](const network::ResourceRequest& request) {
        last_resource_request = request;
      }));

  SetFakeResponse(SerializedResponseProto("Peek Text", DefaultClusters()));

  MockClustersCallback callback;
  SendAndAwaitResponse(GURL("http://www.article.com/"), &callback);

  int load_flags = last_resource_request.load_flags;
  EXPECT_EQ(0, load_flags & net::LOAD_DO_NOT_SEND_COOKIES);
}

TEST_F(ContextualSuggestionsFetcherTest, CookiesExcludedWhenConsentIsDisabled) {
  consent_helper()->SetIsEnabled(false);
  network::ResourceRequest last_resource_request;

  test_factory()->SetInterceptor(
      base::BindLambdaForTesting([&](const network::ResourceRequest& request) {
        last_resource_request = request;
      }));

  SetFakeResponse(SerializedResponseProto("Peek Text", DefaultClusters()));

  MockClustersCallback callback;
  SendAndAwaitResponse(GURL("http://www.article.com/"), &callback);

  int load_flags = last_resource_request.load_flags;
  EXPECT_EQ(net::LOAD_DO_NOT_SEND_COOKIES,
            load_flags & net::LOAD_DO_NOT_SEND_COOKIES);
}

TEST_F(ContextualSuggestionsFetcherTest, ProtocolError) {
  MockClustersCallback callback;
  MockMetricsCallback metrics_callback;
  base::HistogramTester histogram_tester;

  SetFakeResponse("", net::HTTP_NOT_FOUND);
  SendAndAwaitResponse(GURL("http://www.article.com"), &callback,
                       &metrics_callback);

  EXPECT_TRUE(callback.has_run);
  EXPECT_EQ(callback.response_clusters.size(), 0u);
  EXPECT_THAT(
      histogram_tester.GetAllSamples("ContextualSuggestions.FetchResponseCode"),
      ElementsAre(base::Bucket(/*min=*/net::HTTP_NOT_FOUND, /*count=*/1)));
  EXPECT_EQ(metrics_callback.events,
            std::vector<ContextualSuggestionsEvent>(
                {contextual_suggestions::FETCH_ERROR}));
}

TEST_F(ContextualSuggestionsFetcherTest, ServerUnavailable) {
  MockClustersCallback callback;
  MockMetricsCallback metrics_callback;
  base::HistogramTester histogram_tester;

  SetFakeResponse("", net::HTTP_SERVICE_UNAVAILABLE);
  SendAndAwaitResponse(GURL("http://www.article.com"), &callback,
                       &metrics_callback);

  EXPECT_TRUE(callback.has_run);
  EXPECT_EQ(callback.response_clusters.size(), 0u);
  EXPECT_THAT(
      histogram_tester.GetAllSamples("ContextualSuggestions.FetchResponseCode"),
      ElementsAre(base::Bucket(/*min=*/net::HTTP_SERVICE_UNAVAILABLE,
                               /*count=*/1)));
  EXPECT_EQ(metrics_callback.events,
            std::vector<ContextualSuggestionsEvent>(
                {contextual_suggestions::FETCH_SERVER_BUSY}));
}

TEST_F(ContextualSuggestionsFetcherTest, NetworkError) {
  MockClustersCallback callback;
  MockMetricsCallback metrics_callback;
  base::HistogramTester histogram_tester;

  SetFakeResponse(
      "", net::HTTP_OK,
      network::URLLoaderCompletionStatus(net::ERR_CERT_COMMON_NAME_INVALID));
  SendAndAwaitResponse(GURL("http://www.article.com"), &callback,
                       &metrics_callback);

  EXPECT_TRUE(callback.has_run);
  EXPECT_EQ(callback.response_clusters.size(), 0u);
  EXPECT_THAT(
      histogram_tester.GetAllSamples("ContextualSuggestions.FetchErrorCode"),
      ElementsAre(base::Bucket(
          /*min=*/net::ERR_CERT_COMMON_NAME_INVALID, /*count=*/1)));

  EXPECT_EQ(metrics_callback.events,
            std::vector<ContextualSuggestionsEvent>(
                {contextual_suggestions::FETCH_ERROR}));
}

TEST_F(ContextualSuggestionsFetcherTest, EmptyResponse) {
  MockClustersCallback callback;
  MockMetricsCallback metrics_callback;
  SetFakeResponse(SerializedResponseProto("", Cluster()));
  SendAndAwaitResponse(GURL("http://www.article.com/"), &callback,
                       &metrics_callback);

  EXPECT_TRUE(callback.has_run);
  EXPECT_EQ(callback.response_clusters.size(), 0u);

  EXPECT_EQ(metrics_callback.events,
            std::vector<ContextualSuggestionsEvent>(
                {contextual_suggestions::FETCH_EMPTY}));
}

TEST_F(ContextualSuggestionsFetcherTest, ResponseWithUnsetFields) {
  GetPivotsResponse response;
  Pivots* pivots = response.mutable_pivots();
  pivots->add_item()->mutable_document();
  pivots->add_item();

  SetFakeResponse(" " + response.SerializeAsString());
  MockClustersCallback callback;
  SendAndAwaitResponse(GURL("http://www.article.com/"), &callback);

  std::vector<Cluster> expected_clusters;
  expected_clusters.emplace_back(
      ClusterBuilder("")
          .AddSuggestion(SuggestionBuilder(GURL()).Build())
          .AddSuggestion(SuggestionBuilder(GURL()).Build())
          .Build());

  EXPECT_TRUE(callback.has_run);
  EXPECT_EQ(callback.response_clusters.size(), 1u);
  ExpectResponsesMatch(
      std::move(callback),
      ContextualSuggestionsResult("", std::move(expected_clusters),
                                  PeekConditions(), ServerExperimentInfos()));
}

TEST_F(ContextualSuggestionsFetcherTest, CorruptResponse) {
  SetFakeResponse("unparseable proto string");
  MockClustersCallback callback;
  SendAndAwaitResponse(GURL("http://www.article.com/"), &callback);

  EXPECT_TRUE(callback.has_run);
  EXPECT_EQ(callback.response_clusters.size(), 0u);
}

}  // namespace contextual_suggestions