# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Generates a loading report.

When executed as a script, takes a trace filename and print the report.
"""

from activity_lens import ActivityLens
from content_classification_lens import ContentClassificationLens
from loading_graph_view import LoadingGraphView
import loading_trace
import metrics
from network_activity_lens import NetworkActivityLens
from prefetch_view import PrefetchSimulationView
from queuing_lens import QueuingLens
import request_dependencies_lens
from user_satisfied_lens import (
    FirstTextPaintLens, FirstContentfulPaintLens, FirstSignificantPaintLens,
    PLTLens)


def _ComputeCpuBusyness(activity, load_start, satisfied_end):
  """Generates a breakdown of CPU activity between |load_start| and
  |satisfied_end|."""
  duration = float(satisfied_end - load_start)
  result = {
      'activity_frac': (
          activity.MainRendererThreadBusyness(load_start, satisfied_end)
          / duration),
  }

  activity_breakdown = activity.ComputeActivity(load_start, satisfied_end)
  result['parsing_frac'] = (
      sum(activity_breakdown['parsing'].values()) / duration)
  result['script_frac'] = (
      sum(activity_breakdown['script'].values()) / duration)
  return result


class PerUserLensReport(object):
  """Generates a variety of metrics relative to a passed in user lens."""

  def __init__(self, trace, user_lens, activity_lens, network_lens,
               navigation_start_msec):
    requests = trace.request_track.GetEvents()
    dependencies_lens = request_dependencies_lens.RequestDependencyLens(
        trace)
    prefetch_view = PrefetchSimulationView(trace, dependencies_lens, user_lens)
    preloaded_requests = prefetch_view.PreloadedRequests(
        requests[0], dependencies_lens, trace)

    self._navigation_start_msec = navigation_start_msec

    self._satisfied_msec = user_lens.SatisfiedMs()

    graph = LoadingGraphView.FromTrace(trace)
    self._inversions = graph.GetInversionsAtTime(self._satisfied_msec)

    self._byte_frac = self._GenerateByteFrac(network_lens)

    self._requests = user_lens.CriticalRequests()
    self._preloaded_requests = (
        [r for r in preloaded_requests if r in self._requests])

    self._cpu_busyness = _ComputeCpuBusyness(activity_lens,
                                             navigation_start_msec,
                                             self._satisfied_msec)
    prefetch_view.UpdateNodeCosts(lambda n: 0 if n.preloaded else n.cost)
    self._no_state_prefetch_ms = prefetch_view.Cost()

  def GenerateReport(self):
    report = {}

    report['ms'] = self._satisfied_msec - self._navigation_start_msec
    report['byte_frac'] = self._byte_frac

    report['requests'] = len(self._requests)
    report['preloaded_requests'] = len(self._preloaded_requests)
    report['requests_cost'] = reduce(lambda x,y: x + y.Cost(),
                                     self._requests, 0)
    report['preloaded_requests_cost'] = reduce(lambda x,y: x + y.Cost(),
                                        self._preloaded_requests, 0)
    report['predicted_no_state_prefetch_ms'] = self._no_state_prefetch_ms

    # Take the first (earliest) inversion.
    report['inversion'] = self._inversions[0].url if self._inversions else ''

    report.update(self._cpu_busyness)
    return report

  def _GenerateByteFrac(self, network_lens):
    if not network_lens.total_download_bytes:
      return float('Nan')
    byte_frac = (network_lens.DownloadedBytesAt(self._satisfied_msec)
          / float(network_lens.total_download_bytes))
    return byte_frac


class LoadingReport(object):
  """Generates a loading report from a loading trace."""
  def __init__(self, trace, ad_rules=None, tracking_rules=None):
    """Constructor.

    Args:
      trace: (LoadingTrace) a loading trace.
      ad_rules: ([str]) List of ad filtering rules.
      tracking_rules: ([str]) List of tracking filtering rules.
    """
    self.trace = trace

    navigation_start_events = trace.tracing_track.GetMatchingEvents(
        'blink.user_timing', 'navigationStart')
    self._navigation_start_msec = min(
        e.start_msec for e in navigation_start_events)

    self._dns_requests, self._dns_cost_msec = metrics.DnsRequestsAndCost(trace)
    self._connection_stats = metrics.ConnectionMetrics(trace)

    self._user_lens_reports = {}
    plt_lens = PLTLens(self.trace)
    first_text_paint_lens = FirstTextPaintLens(self.trace)
    first_contentful_paint_lens = FirstContentfulPaintLens(self.trace)
    first_significant_paint_lens = FirstSignificantPaintLens(self.trace)
    activity = ActivityLens(trace)
    network_lens = NetworkActivityLens(self.trace)
    for key, user_lens in [['plt', plt_lens],
                           ['first_text', first_text_paint_lens],
                           ['contentful', first_contentful_paint_lens],
                           ['significant', first_significant_paint_lens]]:
      self._user_lens_reports[key] = PerUserLensReport(self.trace,
          user_lens, activity, network_lens, self._navigation_start_msec)

    self._transfer_size = metrics.TotalTransferSize(trace)[1]
    self._request_count = len(trace.request_track.GetEvents())

    content_lens = ContentClassificationLens(
        trace, ad_rules or [], tracking_rules or [])
    has_ad_rules = bool(ad_rules)
    has_tracking_rules = bool(tracking_rules)
    self._ad_report = self._AdRequestsReport(
        trace, content_lens, has_ad_rules, has_tracking_rules)
    self._ads_cost = self._AdsAndTrackingCpuCost(
        self._navigation_start_msec,
        (self._navigation_start_msec
         + self._user_lens_reports['plt'].GenerateReport()['ms']),
        content_lens, activity, has_tracking_rules or has_ad_rules)

    self._queue_stats = self._ComputeQueueStats(QueuingLens(trace))

  def GenerateReport(self):
    """Returns a report as a dict."""
    # NOTE: When changing the return value here, also update the schema
    # (bigquery_schema.json) accordingly. See cloud/frontend/README.md for
    # details.
    report = {
        'url': self.trace.url,
        'transfer_size': self._transfer_size,
        'dns_requests': self._dns_requests,
        'dns_cost_ms': self._dns_cost_msec,
        'total_requests': self._request_count}

    for user_lens_type, user_lens_report in self._user_lens_reports.iteritems():
      for key, value in user_lens_report.GenerateReport().iteritems():
        report[user_lens_type + '_' + key] = value

    report.update(self._ad_report)
    report.update(self._ads_cost)
    report.update(self._connection_stats)
    report.update(self._queue_stats)
    return report

  @classmethod
  def FromTraceFilename(cls, filename, ad_rules_filename,
                        tracking_rules_filename):
    """Returns a LoadingReport from a trace filename."""
    trace = loading_trace.LoadingTrace.FromJsonFile(filename)
    return LoadingReport(trace, ad_rules_filename, tracking_rules_filename)

  @classmethod
  def _AdRequestsReport(
      cls, trace, content_lens, has_ad_rules, has_tracking_rules):
    requests = trace.request_track.GetEvents()
    has_rules = has_ad_rules or has_tracking_rules
    result = {
        'ad_requests': 0 if has_ad_rules else None,
        'tracking_requests': 0 if has_tracking_rules else None,
        'ad_or_tracking_requests': 0 if has_rules else None,
        'ad_or_tracking_initiated_requests': 0 if has_rules else None,
        'ad_or_tracking_initiated_transfer_size': 0 if has_rules else None}
    if not has_rules:
      return result
    for request in requests:
      is_ad = content_lens.IsAdRequest(request)
      is_tracking = content_lens.IsTrackingRequest(request)
      if has_ad_rules:
        result['ad_requests'] += int(is_ad)
      if has_tracking_rules:
        result['tracking_requests'] += int(is_tracking)
      result['ad_or_tracking_requests'] += int(is_ad or is_tracking)
    ad_tracking_requests = content_lens.AdAndTrackingRequests()
    result['ad_or_tracking_initiated_requests'] = len(ad_tracking_requests)
    result['ad_or_tracking_initiated_transfer_size'] = metrics.TransferSize(
        ad_tracking_requests)[1]
    return result

  @classmethod
  def _ComputeQueueStats(cls, queue_lens):
    queuing_info = queue_lens.GenerateRequestQueuing()
    total_blocked_msec = 0
    total_loading_msec = 0
    num_blocking_requests = []
    for queue_info in queuing_info.itervalues():
      try:
        total_blocked_msec += max(0, queue_info.ready_msec -
                                  queue_info.start_msec)
        total_loading_msec += max(0, queue_info.end_msec -
                                  queue_info.start_msec)
      except TypeError:
        pass  # Invalid queue info timings.
      num_blocking_requests.append(len(queue_info.blocking))
    if num_blocking_requests:
      num_blocking_requests.sort()
      avg_blocking = (float(sum(num_blocking_requests)) /
                      len(num_blocking_requests))
      mid = len(num_blocking_requests) / 2
      if len(num_blocking_requests) & 1:
        median_blocking = num_blocking_requests[mid]
      else:
        median_blocking = (num_blocking_requests[mid-1] +
                           num_blocking_requests[mid]) / 2
    else:
      avg_blocking = 0
      median_blocking = 0
    return {
        'total_queuing_blocked_msec': int(total_blocked_msec),
        'total_queuing_load_msec': int(total_loading_msec),
        'average_blocking_request_count': avg_blocking,
        'median_blocking_request_count': median_blocking,
    }

  @classmethod
  def _AdsAndTrackingCpuCost(
      cls, start_msec, end_msec, content_lens, activity, has_rules):
    """Returns the CPU cost associated with Ads and tracking between timestamps.

    Can return an overestimate, as execution slices are tagged by URL, and not
    by requests.

    Args:
      start_msec: (float)
      end_msec: (float)
      content_lens: (ContentClassificationLens)
      activity: (ActivityLens)

    Returns:
      {'ad_and_tracking_script_frac': float,
       'ad_and_tracking_parsing_frac': float}
    """
    result = {'ad_or_tracking_script_frac': None,
              'ad_or_tracking_parsing_frac': None}
    if not has_rules:
      return result

    duration = float(end_msec - start_msec)
    requests = content_lens.AdAndTrackingRequests()
    urls = {r.url for r in requests}
    cpu_breakdown = activity.ComputeActivity(start_msec, end_msec)
    result['ad_or_tracking_script_frac'] = sum(
            value for (url, value) in cpu_breakdown['script'].items()
            if url in urls) / duration
    result['ad_or_tracking_parsing_frac'] = sum(
            value for (url, value) in cpu_breakdown['parsing'].items()
            if url in urls) / duration
    return result


def _Main(args):
  if len(args) not in (2, 4):
    print 'Usage: report.py trace.json (ad_rules tracking_rules)'
    sys.exit(1)
  trace_filename = args[1]
  ad_rules = None
  tracking_rules = None
  if len(args) == 4:
    ad_rules = open(args[2]).readlines()
    tracking_rules = open(args[3]).readlines()
  report = LoadingReport.FromTraceFilename(
      trace_filename, ad_rules, tracking_rules)
  print json.dumps(report.GenerateReport(), indent=2, sort_keys=True)


if __name__ == '__main__':
  import sys
  import json

  _Main(sys.argv)