| // Copyright 2013 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "components/dom_distiller/core/distiller.h" |
| |
| #include <map> |
| #include <memory> |
| #include <utility> |
| #include <vector> |
| |
| #include "base/auto_reset.h" |
| #include "base/bind.h" |
| #include "base/callback.h" |
| #include "base/location.h" |
| #include "base/memory/ptr_util.h" |
| #include "base/metrics/histogram_macros.h" |
| #include "base/single_thread_task_runner.h" |
| #include "base/strings/string_number_conversions.h" |
| #include "base/strings/utf_string_conversions.h" |
| #include "base/threading/thread_task_runner_handle.h" |
| #include "base/values.h" |
| #include "build/build_config.h" |
| #include "components/dom_distiller/core/distiller_page.h" |
| #include "components/dom_distiller/core/distiller_url_fetcher.h" |
| #include "components/dom_distiller/core/proto/distilled_article.pb.h" |
| #include "components/dom_distiller/core/proto/distilled_page.pb.h" |
| #include "net/url_request/url_request_context_getter.h" |
| |
| namespace { |
| // Maximum number of distilled pages in an article. |
| const size_t kMaxPagesInArticle = 32; |
| } |
| |
| namespace dom_distiller { |
| |
| DistillerFactoryImpl::DistillerFactoryImpl( |
| std::unique_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory, |
| const dom_distiller::proto::DomDistillerOptions& dom_distiller_options) |
| : distiller_url_fetcher_factory_(std::move(distiller_url_fetcher_factory)), |
| dom_distiller_options_(dom_distiller_options) {} |
| |
| DistillerFactoryImpl::~DistillerFactoryImpl() {} |
| |
| std::unique_ptr<Distiller> DistillerFactoryImpl::CreateDistillerForUrl( |
| const GURL& unused) { |
| // This default implementation has the same behavior for all URLs. |
| std::unique_ptr<DistillerImpl> distiller(new DistillerImpl( |
| *distiller_url_fetcher_factory_, dom_distiller_options_)); |
| return std::move(distiller); |
| } |
| |
| DistillerImpl::DistilledPageData::DistilledPageData() {} |
| |
| DistillerImpl::DistilledPageData::~DistilledPageData() {} |
| |
| DistillerImpl::DistillerImpl( |
| const DistillerURLFetcherFactory& distiller_url_fetcher_factory, |
| const dom_distiller::proto::DomDistillerOptions& dom_distiller_options) |
| : distiller_url_fetcher_factory_(distiller_url_fetcher_factory), |
| dom_distiller_options_(dom_distiller_options), |
| max_pages_in_article_(kMaxPagesInArticle), |
| destruction_allowed_(true), |
| weak_factory_(this) { |
| } |
| |
| DistillerImpl::~DistillerImpl() { |
| DCHECK(destruction_allowed_); |
| } |
| |
| bool DistillerImpl::DoesFetchImages() { |
| // Only iOS makes use of the fetched image data. |
| #if defined(OS_IOS) |
| return true; |
| #else |
| return false; |
| #endif |
| } |
| |
| void DistillerImpl::SetMaxNumPagesInArticle(size_t max_num_pages) { |
| max_pages_in_article_ = max_num_pages; |
| } |
| |
| bool DistillerImpl::AreAllPagesFinished() const { |
| return started_pages_index_.empty() && waiting_pages_.empty(); |
| } |
| |
| size_t DistillerImpl::TotalPageCount() const { |
| return waiting_pages_.size() + started_pages_index_.size() + |
| finished_pages_index_.size(); |
| } |
| |
| void DistillerImpl::AddToDistillationQueue(int page_num, const GURL& url) { |
| if (!IsPageNumberInUse(page_num) && url.is_valid() && |
| TotalPageCount() < max_pages_in_article_ && |
| seen_urls_.find(url.spec()) == seen_urls_.end()) { |
| waiting_pages_[page_num] = url; |
| } |
| } |
| |
| bool DistillerImpl::IsPageNumberInUse(int page_num) const { |
| return waiting_pages_.find(page_num) != waiting_pages_.end() || |
| started_pages_index_.find(page_num) != started_pages_index_.end() || |
| finished_pages_index_.find(page_num) != finished_pages_index_.end(); |
| } |
| |
| DistillerImpl::DistilledPageData* DistillerImpl::GetPageAtIndex(size_t index) |
| const { |
| DCHECK_LT(index, pages_.size()); |
| DistilledPageData* page_data = pages_[index].get(); |
| DCHECK(page_data); |
| return page_data; |
| } |
| |
| void DistillerImpl::DistillPage(const GURL& url, |
| std::unique_ptr<DistillerPage> distiller_page, |
| const DistillationFinishedCallback& finished_cb, |
| const DistillationUpdateCallback& update_cb) { |
| DCHECK(AreAllPagesFinished()); |
| distiller_page_ = std::move(distiller_page); |
| finished_cb_ = finished_cb; |
| update_cb_ = update_cb; |
| |
| AddToDistillationQueue(0, url); |
| DistillNextPage(); |
| } |
| |
| void DistillerImpl::DistillNextPage() { |
| if (!waiting_pages_.empty()) { |
| auto front = waiting_pages_.begin(); |
| int page_num = front->first; |
| const GURL url = front->second; |
| |
| waiting_pages_.erase(front); |
| DCHECK(url.is_valid()); |
| DCHECK(started_pages_index_.find(page_num) == started_pages_index_.end()); |
| DCHECK(finished_pages_index_.find(page_num) == finished_pages_index_.end()); |
| seen_urls_.insert(url.spec()); |
| pages_.push_back(std::make_unique<DistilledPageData>()); |
| started_pages_index_[page_num] = pages_.size() - 1; |
| distiller_page_->DistillPage( |
| url, |
| dom_distiller_options_, |
| base::Bind(&DistillerImpl::OnPageDistillationFinished, |
| weak_factory_.GetWeakPtr(), |
| page_num, |
| url)); |
| } |
| } |
| |
| void DistillerImpl::OnPageDistillationFinished( |
| int page_num, |
| const GURL& page_url, |
| std::unique_ptr<proto::DomDistillerResult> distiller_result, |
| bool distillation_successful) { |
| DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end()); |
| if (!distillation_successful) { |
| started_pages_index_.erase(page_num); |
| RunDistillerCallbackIfDone(); |
| return; |
| } |
| |
| if (distiller_result->has_statistics_info() && page_num == 0) { |
| if (distiller_result->statistics_info().has_word_count()) { |
| UMA_HISTOGRAM_CUSTOM_COUNTS( |
| "DomDistiller.Statistics.FirstPageWordCount", |
| distiller_result->statistics_info().word_count(), |
| 1, 4000, 50); |
| } |
| } |
| |
| DCHECK(distiller_result); |
| DistilledPageData* page_data = |
| GetPageAtIndex(started_pages_index_[page_num]); |
| page_data->distilled_page_proto = |
| new base::RefCountedData<DistilledPageProto>(); |
| page_data->page_num = page_num; |
| if (distiller_result->has_title()) { |
| page_data->distilled_page_proto->data.set_title( |
| distiller_result->title()); |
| } |
| page_data->distilled_page_proto->data.set_url(page_url.spec()); |
| bool content_empty = true; |
| if (distiller_result->has_distilled_content() && |
| distiller_result->distilled_content().has_html()) { |
| page_data->distilled_page_proto->data.set_html( |
| distiller_result->distilled_content().html()); |
| if (!distiller_result->distilled_content().html().empty()) { |
| content_empty = false; |
| } |
| } |
| |
| if (distiller_result->has_timing_info()) { |
| const proto::TimingInfo& distiller_timing_info = |
| distiller_result->timing_info(); |
| DistilledPageProto::TimingInfo timing_info; |
| if (distiller_timing_info.has_markup_parsing_time()) { |
| timing_info.set_name("markup_parsing"); |
| timing_info.set_time(distiller_timing_info.markup_parsing_time()); |
| *page_data->distilled_page_proto->data.add_timing_info() = timing_info; |
| } |
| |
| if (distiller_timing_info.has_document_construction_time()) { |
| timing_info.set_name("document_construction"); |
| timing_info.set_time( |
| distiller_timing_info.document_construction_time()); |
| *page_data->distilled_page_proto->data.add_timing_info() = timing_info; |
| } |
| |
| if (distiller_timing_info.has_article_processing_time()) { |
| timing_info.set_name("article_processing"); |
| timing_info.set_time( |
| distiller_timing_info.article_processing_time()); |
| *page_data->distilled_page_proto->data.add_timing_info() = timing_info; |
| } |
| |
| if (distiller_timing_info.has_formatting_time()) { |
| timing_info.set_name("formatting"); |
| timing_info.set_time( |
| distiller_timing_info.formatting_time()); |
| *page_data->distilled_page_proto->data.add_timing_info() = timing_info; |
| } |
| |
| if (distiller_timing_info.has_total_time()) { |
| timing_info.set_name("total"); |
| timing_info.set_time( |
| distiller_timing_info.total_time()); |
| *page_data->distilled_page_proto->data.add_timing_info() = timing_info; |
| } |
| |
| for (int i = 0; i < distiller_timing_info.other_times_size(); i++) { |
| timing_info.set_name(distiller_timing_info.other_times(i).name()); |
| timing_info.set_time(distiller_timing_info.other_times(i).time()); |
| *page_data->distilled_page_proto->data.add_timing_info() = timing_info; |
| } |
| } |
| |
| if (distiller_result->has_debug_info() && |
| distiller_result->debug_info().has_log()) { |
| page_data->distilled_page_proto->data.mutable_debug_info()->set_log( |
| distiller_result->debug_info().log()); |
| } |
| |
| if (distiller_result->has_text_direction()) { |
| page_data->distilled_page_proto->data.set_text_direction( |
| distiller_result->text_direction()); |
| } else { |
| page_data->distilled_page_proto->data.set_text_direction("auto"); |
| } |
| |
| if (distiller_result->has_pagination_info()) { |
| const proto::PaginationInfo& pagination_info = |
| distiller_result->pagination_info(); |
| // Skip the next page if the first page is empty. |
| if (pagination_info.has_next_page() && |
| (page_num != 0 || !content_empty)) { |
| GURL next_page_url(pagination_info.next_page()); |
| if (next_page_url.is_valid()) { |
| // The pages should be in same origin. |
| DCHECK_EQ(next_page_url.GetOrigin(), page_url.GetOrigin()); |
| AddToDistillationQueue(page_num + 1, next_page_url); |
| page_data->distilled_page_proto->data.mutable_pagination_info()-> |
| set_next_page(next_page_url.spec()); |
| } |
| } |
| |
| if (pagination_info.has_prev_page()) { |
| GURL prev_page_url(pagination_info.prev_page()); |
| if (prev_page_url.is_valid()) { |
| DCHECK_EQ(prev_page_url.GetOrigin(), page_url.GetOrigin()); |
| AddToDistillationQueue(page_num - 1, prev_page_url); |
| page_data->distilled_page_proto->data.mutable_pagination_info()-> |
| set_prev_page(prev_page_url.spec()); |
| } |
| } |
| |
| if (pagination_info.has_canonical_page()) { |
| GURL canonical_page_url(pagination_info.canonical_page()); |
| if (canonical_page_url.is_valid()) { |
| page_data->distilled_page_proto->data.mutable_pagination_info()-> |
| set_canonical_page(canonical_page_url.spec()); |
| } |
| } |
| } |
| |
| for (int img_num = 0; img_num < distiller_result->content_images_size(); |
| ++img_num) { |
| std::string image_id = base::NumberToString(page_num + 1) + "_" + |
| base::NumberToString(img_num); |
| MaybeFetchImage(page_num, image_id, |
| distiller_result->content_images(img_num).url()); |
| } |
| |
| AddPageIfDone(page_num); |
| DistillNextPage(); |
| } |
| |
| void DistillerImpl::MaybeFetchImage(int page_num, |
| const std::string& image_id, |
| const std::string& image_url) { |
| if (!GURL(image_url).is_valid()) return; |
| DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end()); |
| DistilledPageData* page_data = GetPageAtIndex(started_pages_index_[page_num]); |
| |
| if (!DoesFetchImages()) { |
| DistilledPageProto_Image* image = |
| page_data->distilled_page_proto->data.add_image(); |
| image->set_name(image_id); |
| image->set_url(image_url); |
| return; |
| } |
| |
| DistillerURLFetcher* fetcher = |
| distiller_url_fetcher_factory_.CreateDistillerURLFetcher(); |
| page_data->image_fetchers_.push_back(base::WrapUnique(fetcher)); |
| |
| fetcher->FetchURL(image_url, |
| base::Bind(&DistillerImpl::OnFetchImageDone, |
| weak_factory_.GetWeakPtr(), |
| page_num, |
| base::Unretained(fetcher), |
| image_id, |
| image_url)); |
| } |
| |
| void DistillerImpl::OnFetchImageDone(int page_num, |
| DistillerURLFetcher* url_fetcher, |
| const std::string& id, |
| const std::string& original_url, |
| const std::string& response) { |
| DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end()); |
| DistilledPageData* page_data = GetPageAtIndex(started_pages_index_[page_num]); |
| DCHECK(page_data->distilled_page_proto); |
| DCHECK(url_fetcher); |
| auto fetcher_it = std::find_if( |
| page_data->image_fetchers_.begin(), page_data->image_fetchers_.end(), |
| [url_fetcher](const std::unique_ptr<DistillerURLFetcher>& f) { |
| return url_fetcher == f.get(); |
| }); |
| |
| DCHECK(fetcher_it != page_data->image_fetchers_.end()); |
| // Delete the |url_fetcher| by DeleteSoon since the OnFetchImageDone |
| // callback is invoked by the |url_fetcher|. |
| fetcher_it->release(); |
| page_data->image_fetchers_.erase(fetcher_it); |
| base::ThreadTaskRunnerHandle::Get()->DeleteSoon(FROM_HERE, url_fetcher); |
| |
| DistilledPageProto_Image* image = |
| page_data->distilled_page_proto->data.add_image(); |
| image->set_name(id); |
| image->set_data(response); |
| image->set_url(original_url); |
| |
| AddPageIfDone(page_num); |
| } |
| |
| void DistillerImpl::AddPageIfDone(int page_num) { |
| DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end()); |
| DCHECK(finished_pages_index_.find(page_num) == finished_pages_index_.end()); |
| DistilledPageData* page_data = GetPageAtIndex(started_pages_index_[page_num]); |
| if (page_data->image_fetchers_.empty()) { |
| finished_pages_index_[page_num] = started_pages_index_[page_num]; |
| started_pages_index_.erase(page_num); |
| const ArticleDistillationUpdate& article_update = |
| CreateDistillationUpdate(); |
| DCHECK_EQ(article_update.GetPagesSize(), finished_pages_index_.size()); |
| update_cb_.Run(article_update); |
| RunDistillerCallbackIfDone(); |
| } |
| } |
| |
| const ArticleDistillationUpdate DistillerImpl::CreateDistillationUpdate() |
| const { |
| bool has_prev_page = false; |
| bool has_next_page = false; |
| if (!finished_pages_index_.empty()) { |
| int prev_page_num = finished_pages_index_.begin()->first - 1; |
| int next_page_num = finished_pages_index_.rbegin()->first + 1; |
| has_prev_page = IsPageNumberInUse(prev_page_num); |
| has_next_page = IsPageNumberInUse(next_page_num); |
| } |
| |
| std::vector<scoped_refptr<ArticleDistillationUpdate::RefCountedPageProto> > |
| update_pages; |
| for (auto it = finished_pages_index_.begin(); |
| it != finished_pages_index_.end(); ++it) { |
| update_pages.push_back(pages_[it->second]->distilled_page_proto); |
| } |
| return ArticleDistillationUpdate(update_pages, has_next_page, has_prev_page); |
| } |
| |
| void DistillerImpl::RunDistillerCallbackIfDone() { |
| DCHECK(!finished_cb_.is_null()); |
| if (AreAllPagesFinished()) { |
| bool first_page = true; |
| std::unique_ptr<DistilledArticleProto> article_proto( |
| new DistilledArticleProto()); |
| // Stitch the pages back into the article. |
| for (auto it = finished_pages_index_.begin(); |
| it != finished_pages_index_.end();) { |
| DistilledPageData* page_data = GetPageAtIndex(it->second); |
| *(article_proto->add_pages()) = page_data->distilled_page_proto->data; |
| |
| if (first_page) { |
| article_proto->set_title(page_data->distilled_page_proto->data.title()); |
| first_page = false; |
| } |
| |
| finished_pages_index_.erase(it++); |
| } |
| |
| pages_.clear(); |
| DCHECK_LE(static_cast<size_t>(article_proto->pages_size()), |
| max_pages_in_article_); |
| |
| DCHECK(pages_.empty()); |
| DCHECK(finished_pages_index_.empty()); |
| |
| base::AutoReset<bool> dont_delete_this_in_callback(&destruction_allowed_, |
| false); |
| finished_cb_.Run(std::move(article_proto)); |
| finished_cb_.Reset(); |
| } |
| } |
| |
| } // namespace dom_distiller |
