// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "components/ntp_snippets/ntp_snippets_fetcher.h"

#include <stdlib.h>

#include "base/command_line.h"
#include "base/files/file_path.h"
#include "base/files/file_util.h"
#include "base/metrics/histogram_macros.h"
#include "base/metrics/sparse_histogram.h"
#include "base/path_service.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/string_util.h"
#include "base/strings/stringprintf.h"
#include "base/time/default_tick_clock.h"
#include "base/values.h"
#include "components/data_use_measurement/core/data_use_user_data.h"
#include "components/ntp_snippets/ntp_snippets_constants.h"
#include "components/ntp_snippets/switches.h"
#include "components/signin/core/browser/profile_oauth2_token_service.h"
#include "components/signin/core/browser/signin_manager.h"
#include "components/signin/core/browser/signin_manager_base.h"
#include "components/variations/variations_associated_data.h"
#include "google_apis/google_api_keys.h"
#include "net/base/load_flags.h"
#include "net/http/http_request_headers.h"
#include "net/http/http_response_headers.h"
#include "net/http/http_status_code.h"
#include "net/url_request/url_fetcher.h"
#include "third_party/icu/source/common/unicode/uloc.h"
#include "third_party/icu/source/common/unicode/utypes.h"

using net::URLFetcher;
using net::URLRequestContextGetter;
using net::HttpRequestHeaders;
using net::URLRequestStatus;

namespace ntp_snippets {

namespace {

const char kApiScope[] = "https://www.googleapis.com/auth/webhistory";
const char kSnippetsServer[] =
    "https://chromereader-pa.googleapis.com/v1/fetch";
const char kSnippetsServerNonAuthorizedFormat[] = "%s?key=%s";
const char kAuthorizationRequestHeaderFormat[] = "Bearer %s";

// Variation parameter for personalizing fetching of snippets.
const char kPersonalizationName[] = "fetching_personalization";
// Variation parameter for setting whether to restrict to a passed set of hosts.
const char kHostRestrictionName[] = "fetching_host_restrict";

// Constants for possible values of the "fetching_personalization" parameter.
const char kPersonalizationPersonalString[] = "personal";
const char kPersonalizationNonPersonalString[] = "non_personal";
const char kPersonalizationBothString[] = "both";  // the default value

// Constants for possible values of the "fetching_host_restrict" parameter.
const char kHostRestrictionOnString[] = "on";  // the default value
const char kHostRestrictionOffString[] = "off";

const char kRequestFormat[] =
    "{"
    "  \"response_detail_level\": \"STANDARD\","
    "%s"  // If authenticated - an obfuscated Gaia ID will be inserted here.
    "  \"advanced_options\": {"
    "    \"local_scoring_params\": {"
    "      \"content_params\": {"
    "        \"only_return_personalized_results\": %s"
    "%s"  // If authenticated - user segment (lang code) will be inserted here.
    "      },"
    "      \"content_restricts\": ["
    "        {"
    "          \"type\": \"METADATA\","
    "          \"value\": \"TITLE\""
    "        },"
    "        {"
    "          \"type\": \"METADATA\","
    "          \"value\": \"SNIPPET\""
    "        },"
    "        {"
    "          \"type\": \"METADATA\","
    "          \"value\": \"THUMBNAIL\""
    "        }"
    "      ],"
    "      \"content_selectors\": [%s]"
    "    },"
    "    \"global_scoring_params\": {"
    "      \"num_to_return\": %i,"
    "      \"sort_type\": 1"
    "    }"
    "  }"
    "}";

const char kGaiaIdFormat[] = "  \"obfuscated_gaia_id\": \"%s\",";
const char kUserSegmentFormat[] = "        ,\"user_segment\": \"%s\"";
const char kHostRestrictFormat[] =
    "      {"
    "        \"type\": \"HOST_RESTRICT\","
    "        \"value\": \"%s\""
    "      }";
const char kTrueString[] = "true";
const char kFalseString[] = "false";

std::string FetchResultToString(NTPSnippetsFetcher::FetchResult result) {
  switch (result) {
    case NTPSnippetsFetcher::FetchResult::SUCCESS:
      return "OK";
    case NTPSnippetsFetcher::FetchResult::EMPTY_HOSTS:
      return "Cannot fetch for empty hosts list.";
    case NTPSnippetsFetcher::FetchResult::URL_REQUEST_STATUS_ERROR:
      return "URLRequestStatus error";
    case NTPSnippetsFetcher::FetchResult::HTTP_ERROR:
      return "HTTP error";
    case NTPSnippetsFetcher::FetchResult::JSON_PARSE_ERROR:
      return "Received invalid JSON";
    case NTPSnippetsFetcher::FetchResult::INVALID_SNIPPET_CONTENT_ERROR:
      return "Invalid / empty list.";
    case NTPSnippetsFetcher::FetchResult::OAUTH_TOKEN_ERROR:
      return "Error in obtaining an OAuth2 access token.";
    case NTPSnippetsFetcher::FetchResult::RESULT_MAX:
      break;
  }
  NOTREACHED();
  return "Unknown error";
}

std::string BuildRequest(const std::string& obfuscated_gaia_id,
                         bool only_return_personalized_results,
                         const std::string& user_segment,
                         const std::string& host_restricts,
                         int count_to_fetch) {
  return base::StringPrintf(
      kRequestFormat, obfuscated_gaia_id.c_str(),
      only_return_personalized_results ? kTrueString : kFalseString,
      user_segment.c_str(), host_restricts.c_str(), count_to_fetch);
}

}  // namespace

NTPSnippetsFetcher::NTPSnippetsFetcher(
    SigninManagerBase* signin_manager,
    OAuth2TokenService* token_service,
    scoped_refptr<URLRequestContextGetter> url_request_context_getter,
    const ParseJSONCallback& parse_json_callback,
    bool is_stable_channel)
    : OAuth2TokenService::Consumer("ntp_snippets"),
      signin_manager_(signin_manager),
      token_service_(token_service),
      waiting_for_refresh_token_(false),
      url_request_context_getter_(url_request_context_getter),
      parse_json_callback_(parse_json_callback),
      is_stable_channel_(is_stable_channel),
      tick_clock_(new base::DefaultTickClock()),
      weak_ptr_factory_(this) {
  // Parse the variation parameters and set the defaults if missing.
  std::string personalization = variations::GetVariationParamValue(
      ntp_snippets::kStudyName, kPersonalizationName);
  if (personalization == kPersonalizationNonPersonalString) {
    personalization_ = Personalization::kNonPersonal;
  } else if (personalization == kPersonalizationPersonalString) {
    personalization_ = Personalization::kPersonal;
  } else {
    personalization_ = Personalization::kBoth;
    LOG_IF(WARNING, !personalization.empty() &&
                        personalization != kPersonalizationBothString)
        << "Unknown value for " << kPersonalizationName << ": "
        << personalization;
  }

  std::string host_restriction = variations::GetVariationParamValue(
      ntp_snippets::kStudyName, kHostRestrictionName);
  if (host_restriction == kHostRestrictionOffString) {
    use_host_restriction_ = false;
  } else {
    use_host_restriction_ = true;
    LOG_IF(WARNING, !host_restriction.empty() &&
                        host_restriction != kHostRestrictionOnString)
        << "Unknown value for " << kHostRestrictionName << ": "
        << host_restriction;
  }
}

NTPSnippetsFetcher::~NTPSnippetsFetcher() {
  if (waiting_for_refresh_token_)
    token_service_->RemoveObserver(this);
}

void NTPSnippetsFetcher::SetCallback(
    const SnippetsAvailableCallback& callback) {
  snippets_available_callback_ = callback;
}

void NTPSnippetsFetcher::FetchSnippetsFromHosts(
    const std::set<std::string>& hosts,
    const std::string& language_code,
    int count) {
  hosts_ = hosts;
  fetch_start_time_ = tick_clock_->NowTicks();

  if (UsesHostRestrictions() && hosts_.empty()) {
    FetchFinished(OptionalSnippets(), FetchResult::EMPTY_HOSTS,
                  /*extra_message=*/std::string());
    return;
  }

  // Translate the BCP 47 |language_code| into a posix locale string.
  char locale[ULOC_FULLNAME_CAPACITY];
  UErrorCode error = U_ZERO_ERROR;
  uloc_forLanguageTag(language_code.c_str(), locale, ULOC_FULLNAME_CAPACITY,
                      nullptr, &error);
  DLOG_IF(WARNING, U_ZERO_ERROR != error)
      << "Error in translating language code to a locale string: " << error;
  locale_ = locale;

  count_to_fetch_ = count;

  bool use_authentication = UsesAuthentication();

  if (use_authentication && signin_manager_->IsAuthenticated()) {
    // Signed-in: get OAuth token --> fetch snippets.
    StartTokenRequest();
  } else if (use_authentication && signin_manager_->AuthInProgress()) {
    // Currently signing in: wait for auth to finish (the refresh token) -->
    //     get OAuth token --> fetch snippets.
    if (!waiting_for_refresh_token_) {
      // Wait until we get a refresh token.
      waiting_for_refresh_token_ = true;
      token_service_->AddObserver(this);
    }
  } else {
    // Not signed in: fetch snippets (without authentication).
    FetchSnippetsNonAuthenticated();
  }
}

void NTPSnippetsFetcher::FetchSnippetsImpl(const GURL& url,
                                           const std::string& auth_header,
                                           const std::string& request) {
  url_fetcher_ = URLFetcher::Create(url, URLFetcher::POST, this);

  url_fetcher_->SetRequestContext(url_request_context_getter_.get());
  url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES |
                             net::LOAD_DO_NOT_SAVE_COOKIES);

  data_use_measurement::DataUseUserData::AttachToFetcher(
      url_fetcher_.get(), data_use_measurement::DataUseUserData::NTP_SNIPPETS);

  HttpRequestHeaders headers;
  if (!auth_header.empty())
    headers.SetHeader("Authorization", auth_header);
  headers.SetHeader("Content-Type", "application/json; charset=UTF-8");
  url_fetcher_->SetExtraRequestHeaders(headers.ToString());
  url_fetcher_->SetUploadData("application/json", request);
  // Fetchers are sometimes cancelled because a network change was detected.
  url_fetcher_->SetAutomaticallyRetryOnNetworkChanges(3);
  // Try to make fetching the files bit more robust even with poor connection.
  url_fetcher_->SetMaxRetriesOn5xx(3);
  url_fetcher_->Start();
}

std::string NTPSnippetsFetcher::GetHostRestricts() const {
  std::string host_restricts;
  if (UsesHostRestrictions()) {
    for (const std::string& host : hosts_) {
      if (!host_restricts.empty())
        host_restricts.push_back(',');
      host_restricts += base::StringPrintf(kHostRestrictFormat, host.c_str());
    }
  }
  return host_restricts;
}

bool NTPSnippetsFetcher::UsesHostRestrictions() const {
  return use_host_restriction_ &&
         !base::CommandLine::ForCurrentProcess()->HasSwitch(
             switches::kDontRestrict);
}

bool NTPSnippetsFetcher::UsesAuthentication() const {
  return (personalization_ == Personalization::kPersonal ||
          personalization_ == Personalization::kBoth);
}

void NTPSnippetsFetcher::FetchSnippetsNonAuthenticated() {
  // When not providing OAuth token, we need to pass the Google API key.
  const std::string& key = is_stable_channel_
                               ? google_apis::GetAPIKey()
                               : google_apis::GetNonStableAPIKey();
  GURL url(base::StringPrintf(kSnippetsServerNonAuthorizedFormat,
                              kSnippetsServer, key.c_str()));

  FetchSnippetsImpl(url, std::string(),
                    BuildRequest(/*obfuscated_gaia_id=*/std::string(),
                                 /*only_return_personalized_results=*/false,
                                 /*user_segment=*/std::string(),
                                 GetHostRestricts(), count_to_fetch_));
}

void NTPSnippetsFetcher::FetchSnippetsAuthenticated(
    const std::string& account_id,
    const std::string& oauth_access_token) {
  std::string gaia_id = base::StringPrintf(kGaiaIdFormat, account_id.c_str());
  std::string user_segment =
      base::StringPrintf(kUserSegmentFormat, locale_.c_str());

  FetchSnippetsImpl(
      GURL(kSnippetsServer),
      base::StringPrintf(kAuthorizationRequestHeaderFormat,
                         oauth_access_token.c_str()),
      BuildRequest(gaia_id, personalization_ == Personalization::kPersonal,
                   user_segment, GetHostRestricts(), count_to_fetch_));
}

void NTPSnippetsFetcher::StartTokenRequest() {
  OAuth2TokenService::ScopeSet scopes;
  scopes.insert(kApiScope);
  oauth_request_ = token_service_->StartRequest(
      signin_manager_->GetAuthenticatedAccountId(), scopes, this);
}

////////////////////////////////////////////////////////////////////////////////
// OAuth2TokenService::Consumer overrides
void NTPSnippetsFetcher::OnGetTokenSuccess(
    const OAuth2TokenService::Request* request,
    const std::string& access_token,
    const base::Time& expiration_time) {
  // Delete the request after we leave this method.
  std::unique_ptr<OAuth2TokenService::Request> oauth_request(
      std::move(oauth_request_));
  DCHECK_EQ(oauth_request.get(), request)
      << "Got tokens from some previous request";

  FetchSnippetsAuthenticated(oauth_request->GetAccountId(), access_token);
}

void NTPSnippetsFetcher::OnGetTokenFailure(
    const OAuth2TokenService::Request* request,
    const GoogleServiceAuthError& error) {
  oauth_request_.reset();
  DLOG(ERROR) << "Unable to get token: " << error.ToString()
              << " - fetching the snippets without authentication.";
  FetchFinished(
      OptionalSnippets(), FetchResult::OAUTH_TOKEN_ERROR,
      /*extra_message=*/base::StringPrintf(" (%s)", error.ToString().c_str()));
}

////////////////////////////////////////////////////////////////////////////////
// OAuth2TokenService::Observer overrides
void NTPSnippetsFetcher::OnRefreshTokenAvailable(
    const std::string& account_id) {
  // Only react on tokens for the account the user has signed in with.
  if (account_id != signin_manager_->GetAuthenticatedAccountId())
    return;

  token_service_->RemoveObserver(this);
  waiting_for_refresh_token_ = false;
  StartTokenRequest();
}

////////////////////////////////////////////////////////////////////////////////
// URLFetcherDelegate overrides
void NTPSnippetsFetcher::OnURLFetchComplete(const URLFetcher* source) {
  DCHECK_EQ(url_fetcher_.get(), source);

  const URLRequestStatus& status = source->GetStatus();

  UMA_HISTOGRAM_SPARSE_SLOWLY(
      "NewTabPage.Snippets.FetchHttpResponseOrErrorCode",
      status.is_success() ? source->GetResponseCode() : status.error());

  if (!status.is_success()) {
    FetchFinished(OptionalSnippets(), FetchResult::URL_REQUEST_STATUS_ERROR,
                  /*extra_message=*/base::StringPrintf(" %d", status.error()));
  } else if (source->GetResponseCode() != net::HTTP_OK) {
    // TODO(jkrcal): https://crbug.com/609084
    // We need to deal with the edge case again where the auth
    // token expires just before we send the request (in which case we need to
    // fetch a new auth token). We should extract that into a common class
    // instead of adding it to every single class that uses auth tokens.
    FetchFinished(
        OptionalSnippets(), FetchResult::HTTP_ERROR,
        /*extra_message=*/base::StringPrintf(" %d", source->GetResponseCode()));
  } else {
    bool stores_result_to_string = source->GetResponseAsString(
        &last_fetch_json_);
    DCHECK(stores_result_to_string);

    parse_json_callback_.Run(
        last_fetch_json_,
        base::Bind(&NTPSnippetsFetcher::OnJsonParsed,
                   weak_ptr_factory_.GetWeakPtr()),
        base::Bind(&NTPSnippetsFetcher::OnJsonError,
                   weak_ptr_factory_.GetWeakPtr()));
  }
}

void NTPSnippetsFetcher::OnJsonParsed(std::unique_ptr<base::Value> parsed) {
  const base::DictionaryValue* top_dict = nullptr;
  const base::ListValue* list = nullptr;
  NTPSnippet::PtrVector snippets;
  if (!parsed->GetAsDictionary(&top_dict) ||
      !top_dict->GetList("recos", &list) ||
      !NTPSnippet::AddFromListValue(*list, &snippets)) {
    LOG(WARNING) << "Received invalid snippets: " << last_fetch_json_;
    FetchFinished(OptionalSnippets(),
                  FetchResult::INVALID_SNIPPET_CONTENT_ERROR,
                  /*extra_message=*/std::string());
  } else {
    FetchFinished(OptionalSnippets(std::move(snippets)), FetchResult::SUCCESS,
                  /*extra_message=*/std::string());
  }
}

void NTPSnippetsFetcher::OnJsonError(const std::string& error) {
  LOG(WARNING) << "Received invalid JSON (" << error << "): "
               << last_fetch_json_;
  FetchFinished(
      OptionalSnippets(), FetchResult::JSON_PARSE_ERROR,
      /*extra_message=*/base::StringPrintf(" (error %s)", error.c_str()));
}

void NTPSnippetsFetcher::FetchFinished(OptionalSnippets snippets,
                                       FetchResult result,
                                       const std::string& extra_message) {
  DCHECK(result == FetchResult::SUCCESS || !snippets);
  last_status_ = FetchResultToString(result) + extra_message;

  UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime",
                      tick_clock_->NowTicks() - fetch_start_time_);
  UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult",
                            static_cast<int>(result),
                            static_cast<int>(FetchResult::RESULT_MAX));

  if (!snippets_available_callback_.is_null())
    snippets_available_callback_.Run(std::move(snippets));
}

}  // namespace ntp_snippets