| # Copyright (c) 2016 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """The request data track. |
| |
| When executed, parses a JSON dump of DevTools messages. |
| """ |
| |
| import bisect |
| import collections |
| import copy |
| import datetime |
| import email.utils |
| import hashlib |
| import json |
| import logging |
| import re |
| import sys |
| import urlparse |
| |
| import devtools_monitor |
| |
| |
| class Timing(object): |
| """Collects the timing data for a request.""" |
| UNVAILABLE = -1 |
| _TIMING_NAMES = ( |
| ('connectEnd', 'connect_end'), ('connectStart', 'connect_start'), |
| ('dnsEnd', 'dns_end'), ('dnsStart', 'dns_start'), |
| ('proxyEnd', 'proxy_end'), ('proxyStart', 'proxy_start'), |
| ('receiveHeadersEnd', 'receive_headers_end'), |
| ('requestTime', 'request_time'), ('sendEnd', 'send_end'), |
| ('sendStart', 'send_start'), ('sslEnd', 'ssl_end'), |
| ('sslStart', 'ssl_start'), ('workerReady', 'worker_ready'), |
| ('workerStart', 'worker_start'), |
| ('loadingFinished', 'loading_finished'), ('pushStart', 'push_start'), |
| ('pushEnd', 'push_end')) |
| _TIMING_NAMES_MAPPING = dict(_TIMING_NAMES) |
| __slots__ = tuple(x[1] for x in _TIMING_NAMES) |
| |
| def __init__(self, **kwargs): |
| """Constructor. |
| |
| Initialize with keywords arguments from __slots__. |
| """ |
| for slot in self.__slots__: |
| setattr(self, slot, self.UNVAILABLE) |
| for (attr, value) in kwargs.items(): |
| setattr(self, attr, value) |
| |
| def __eq__(self, o): |
| return all(getattr(self, attr) == getattr(o, attr) |
| for attr in self.__slots__) |
| |
| def __str__(self): |
| return str(self.ToJsonDict()) |
| |
| def LargestOffset(self): |
| """Returns the largest offset in the available timings.""" |
| return max(0, max( |
| getattr(self, attr) for attr in self.__slots__ |
| if attr != 'request_time')) |
| |
| def ToJsonDict(self): |
| return {attr: getattr(self, attr) |
| for attr in self.__slots__ if getattr(self, attr) != -1} |
| |
| @classmethod |
| def FromJsonDict(cls, json_dict): |
| return cls(**json_dict) |
| |
| @classmethod |
| def FromDevToolsDict(cls, json_dict): |
| """Returns an instance of Timing from a dict, as passed by DevTools.""" |
| timing_dict = { |
| cls._TIMING_NAMES_MAPPING[k]: v for (k, v) in json_dict.items()} |
| return cls(**timing_dict) |
| |
| |
| def ShortName(url): |
| """Returns a shortened version of a URL.""" |
| parsed = urlparse.urlparse(url) |
| path = parsed.path |
| hostname = parsed.hostname if parsed.hostname else '?.?.?' |
| if path != '' and path != '/': |
| last_path = parsed.path.split('/')[-1] |
| if len(last_path) < 10: |
| if len(path) < 10: |
| return hostname + '/' + path |
| else: |
| return hostname + '/..' + parsed.path[-10:] |
| else: |
| return hostname + '/..' + last_path[:5] |
| else: |
| return hostname |
| |
| |
| def IntervalBetween(first, second, reason): |
| """Returns the start and end of the inteval between two requests, in ms. |
| |
| This is defined as: |
| - [first.headers, second.start] if reason is 'parser'. This is to account |
| for incremental parsing. |
| - [first.end, second.start] if reason is 'script', 'redirect' or 'other'. |
| |
| Args: |
| first: (Request) First request. |
| second: (Request) Second request. |
| reason: (str) Link between the two requests, in Request.INITIATORS. |
| |
| Returns: |
| (start_msec (float), end_msec (float)), |
| """ |
| assert reason in Request.INITIATORS |
| second_ms = second.timing.request_time * 1000 |
| if reason == 'parser': |
| first_offset_ms = first.timing.receive_headers_end |
| else: |
| first_offset_ms = first.timing.LargestOffset() |
| return (first.timing.request_time * 1000 + first_offset_ms, second_ms) |
| |
| |
| def TimeBetween(first, second, reason): |
| """(end_msec - start_msec), with the values as returned by IntervalBetween(). |
| """ |
| (first_ms, second_ms) = IntervalBetween(first, second, reason) |
| return second_ms - first_ms |
| |
| |
| def TimingAsList(timing): |
| """Transform Timing to a list, eg as is used in JSON output. |
| |
| Args: |
| timing: a Timing. |
| |
| Returns: |
| A list identical to what the eventual JSON output will be (eg, |
| Request.ToJsonDict). |
| """ |
| return json.loads(json.dumps(timing)) |
| |
| |
| class Request(object): |
| """Represents a single request. |
| |
| Generally speaking, fields here closely mirror those documented in |
| third_party/WebKit/Source/devtools/protocol.json. |
| |
| Fields: |
| request_id: (str) unique request ID. Postfixed with _REDIRECT_SUFFIX for |
| redirects. |
| frame_id: (str) unique frame identifier. |
| loader_id: (str) unique frame identifier. |
| document_url: (str) URL of the document this request is loaded for. |
| url: (str) Request URL. |
| protocol: (str) protocol used for the request. |
| method: (str) HTTP method, such as POST or GET. |
| request_headers: (dict) {'header': 'value'} Request headers. |
| response_headers: (dict) {'header': 'value'} Response headers. |
| initial_priority: (str) Initial request priority, in REQUEST_PRIORITIES. |
| timestamp: (float) Request timestamp, in s. |
| wall_time: (float) Request timestamp, UTC timestamp in s. |
| initiator: (dict) Request initiator, in INITIATORS. |
| resource_type: (str) Resource type, in RESOURCE_TYPES |
| served_from_cache: (bool) Whether the request was served from cache. |
| from_disk_cache: (bool) Whether the request was served from the disk cache. |
| from_service_worker: (bool) Whether the request was served by a Service |
| Worker. |
| timing: (Timing) Request timing, extended with loading_finished. |
| status: (int) Response status code. |
| status_text: (str) Response status text received in the status line. |
| encoded_data_length: (int) Total encoded data length. |
| data_chunks: (list) [(offset, encoded_data_length), ...] List of data |
| chunks received, with their offset in ms relative to |
| Timing.requestTime. |
| failed: (bool) Whether the request failed. |
| error_text: (str) User friendly error message when request failed. |
| start_msec: (float) Request start time, in milliseconds from chrome start. |
| end_msec: (float) Request end time, in milliseconds from chrome start. |
| start_msec. |
| """ |
| REQUEST_PRIORITIES = ('VeryLow', 'Low', 'Medium', 'High', 'VeryHigh') |
| RESOURCE_TYPES = ('Document', 'Stylesheet', 'Image', 'Media', 'Font', |
| 'Script', 'TextTrack', 'XHR', 'Fetch', 'EventSource', |
| 'WebSocket', 'Manifest', 'Other') |
| INITIATORS = ('parser', 'script', 'other', 'redirect') |
| INITIATING_REQUEST = 'initiating_request' |
| ORIGINAL_INITIATOR = 'original_initiator' |
| def __init__(self): |
| self.request_id = None |
| self.frame_id = None |
| self.loader_id = None |
| self.document_url = None |
| self.url = None |
| self.protocol = None |
| self.method = None |
| self.mime_type = None |
| self.request_headers = None |
| self.response_headers = None |
| self.initial_priority = None |
| self.timestamp = -1 |
| self.wall_time = -1 |
| self.initiator = None |
| self.resource_type = None |
| self.served_from_cache = False |
| self.from_disk_cache = False |
| self.from_service_worker = False |
| self.timing = None |
| self.status = None |
| self.status_text = None |
| self.response_headers_length = 0 |
| self.encoded_data_length = 0 |
| self.data_chunks = [] |
| self.failed = False |
| self.error_text = None |
| |
| @property |
| def start_msec(self): |
| return self.timing.request_time * 1000 |
| |
| @property |
| def end_msec(self): |
| if self.start_msec is None: |
| return None |
| return self.start_msec + self.timing.LargestOffset() |
| |
| @property |
| def fingerprint(self): |
| h = hashlib.sha256() |
| h.update(self.url) |
| return h.hexdigest()[:10] |
| |
| def _TimestampOffsetFromStartMs(self, timestamp): |
| assert self.timing.request_time != -1 |
| request_time = self.timing.request_time |
| return (timestamp - request_time) * 1000 |
| |
| def ToJsonDict(self): |
| result = copy.deepcopy(self.__dict__) |
| result['timing'] = self.timing.ToJsonDict() if self.timing else {} |
| return result |
| |
| @classmethod |
| def FromJsonDict(cls, data_dict): |
| result = Request() |
| for (k, v) in data_dict.items(): |
| setattr(result, k, v) |
| if not result.response_headers: |
| result.response_headers = {} |
| if result.timing: |
| result.timing = Timing.FromJsonDict(result.timing) |
| else: |
| result.timing = Timing(request_time=result.timestamp) |
| return result |
| |
| def GetResponseTransportLength(self): |
| """Get the total amount of encoded data no matter whether load has finished |
| or not. |
| """ |
| assert self.HasReceivedResponse() |
| assert not self.from_disk_cache and not self.served_from_cache |
| assert self.protocol not in {'about', 'blob', 'data'} |
| if self.timing.loading_finished != Timing.UNVAILABLE: |
| encoded_data_length = self.encoded_data_length |
| else: |
| encoded_data_length = sum( |
| [chunk_size for _, chunk_size in self.data_chunks]) |
| assert encoded_data_length > 0 or len(self.data_chunks) == 0 |
| return encoded_data_length + self.response_headers_length |
| |
| def GetHTTPResponseHeader(self, header_name): |
| """Gets the value of a HTTP response header. |
| |
| Does a case-insensitive search for the header name in the HTTP response |
| headers, in order to support servers that use a wrong capitalization. |
| """ |
| lower_case_name = header_name.lower() |
| result = None |
| for name, value in self.response_headers.iteritems(): |
| if name.lower() == lower_case_name: |
| result = value |
| break |
| return result |
| |
| def SetHTTPResponseHeader(self, header, header_value): |
| """Sets the value of a HTTP response header.""" |
| assert header.islower() |
| for name in self.response_headers.keys(): |
| if name.lower() == header: |
| del self.response_headers[name] |
| self.response_headers[header] = header_value |
| |
| def GetResponseHeaderValue(self, header, value): |
| """Returns a copy of |value| iff response |header| contains it.""" |
| header_values = self.GetHTTPResponseHeader(header) |
| if not header_values: |
| return None |
| values = header_values.split(',') |
| for header_value in values: |
| if header_value.lower() == value.lower(): |
| return header_value |
| return None |
| |
| def HasResponseHeaderValue(self, header, value): |
| """Returns True iff the response headers |header| contains |value|.""" |
| return self.GetResponseHeaderValue(header, value) is not None |
| |
| def GetContentType(self): |
| """Returns the content type, or None.""" |
| # Check for redirects. Use the "Location" header, because the HTTP status is |
| # not reliable. |
| if self.GetHTTPResponseHeader('Location') is not None: |
| return 'redirect' |
| |
| # Check if the response is empty. |
| if (self.GetHTTPResponseHeader('Content-Length') == '0' or |
| self.status == 204): |
| return 'ping' |
| |
| if self.mime_type: |
| return self.mime_type |
| |
| content_type = self.GetHTTPResponseHeader('Content-Type') |
| if not content_type or ';' not in content_type: |
| return content_type |
| else: |
| return content_type[:content_type.index(';')] |
| |
| def IsDataRequest(self): |
| return self.protocol == 'data' |
| |
| def HasReceivedResponse(self): |
| return self.status is not None |
| |
| def GetCacheControlDirective(self, directive_name): |
| """Returns the value of a Cache-Control directive, or None.""" |
| cache_control_str = self.GetHTTPResponseHeader('Cache-Control') |
| if cache_control_str is None: |
| return None |
| directives = [s.strip() for s in cache_control_str.split(',')] |
| for directive in directives: |
| parts = directive.split('=') |
| if len(parts) != 2: |
| continue |
| (name, value) = parts |
| if name == directive_name: |
| return value |
| return None |
| |
| def MaxAge(self): |
| """Returns the max-age of a resource, or -1.""" |
| # TODO(lizeb): Handle the "Expires" header as well. |
| cache_control = {} |
| if not self.response_headers: |
| return -1 |
| |
| cache_control_str = self.GetHTTPResponseHeader('Cache-Control') |
| if cache_control_str is not None: |
| directives = [s.strip() for s in cache_control_str.split(',')] |
| for directive in directives: |
| parts = [s.strip() for s in directive.split('=')] |
| if len(parts) == 1: |
| cache_control[parts[0]] = True |
| else: |
| cache_control[parts[0]] = parts[1] |
| if (u'no-store' in cache_control |
| or u'no-cache' in cache_control |
| or len(cache_control) == 0): |
| return -1 |
| max_age = self.GetCacheControlDirective('max-age') |
| if max_age: |
| return int(max_age) |
| return -1 |
| |
| def Cost(self): |
| """Returns the cost of this request in ms, defined as time between |
| request_time and the latest timing event. |
| """ |
| # All fields in timing are millis relative to request_time. |
| return self.timing.LargestOffset() |
| |
| def GetRawResponseHeaders(self): |
| """Gets the request's raw response headers compatible with |
| net::HttpResponseHeaders's constructor. |
| """ |
| assert not self.IsDataRequest() |
| assert self.HasReceivedResponse() |
| headers = bytes('{} {} {}\x00'.format( |
| self.protocol.upper(), self.status, self.status_text)) |
| for key in sorted(self.response_headers.keys()): |
| headers += (bytes(key.encode('latin-1')) + b': ' + |
| bytes(self.response_headers[key].encode('latin-1')) + b'\x00') |
| return headers |
| |
| def __eq__(self, o): |
| return self.__dict__ == o.__dict__ |
| |
| def __hash__(self): |
| return hash(self.request_id) |
| |
| def __str__(self): |
| return json.dumps(self.ToJsonDict(), sort_keys=True, indent=2) |
| |
| |
| def _ParseStringToInt(string): |
| """Parses a string to an integer like base::StringToInt64(). |
| |
| Returns: |
| Parsed integer. |
| """ |
| string = string.strip() |
| while string: |
| try: |
| parsed_integer = int(string) |
| if parsed_integer > sys.maxint: |
| return sys.maxint |
| if parsed_integer < -sys.maxint - 1: |
| return -sys.maxint - 1 |
| return parsed_integer |
| except ValueError: |
| string = string[:-1] |
| return 0 |
| |
| |
| class CachingPolicy(object): |
| """Represents the caching policy at an arbitrary time for a cached response. |
| """ |
| FETCH = 'FETCH' |
| VALIDATION_NONE = 'VALIDATION_NONE' |
| VALIDATION_SYNC = 'VALIDATION_SYNC' |
| VALIDATION_ASYNC = 'VALIDATION_ASYNC' |
| POLICIES = (FETCH, VALIDATION_NONE, VALIDATION_SYNC, VALIDATION_ASYNC) |
| def __init__(self, request): |
| """Constructor. |
| |
| Args: |
| request: (Request) |
| """ |
| assert request.response_headers is not None |
| self.request = request |
| # This is incorrect, as the timestamp corresponds to when devtools is made |
| # aware of the request, not when it was sent. However, this is good enough |
| # for computing cache expiration, which doesn't need sub-second precision. |
| self._request_time = self.request.wall_time |
| # Used when the date is not available. |
| self._response_time = ( |
| self._request_time + self.request.timing.receive_headers_end) |
| |
| def HasValidators(self): |
| """Returns wether the request has a validator.""" |
| # Assuming HTTP 1.1+. |
| return (self.request.GetHTTPResponseHeader('Last-Modified') |
| or self.request.GetHTTPResponseHeader('Etag')) |
| |
| def IsCacheable(self): |
| """Returns whether the request could be stored in the cache.""" |
| return not self.request.HasResponseHeaderValue('Cache-Control', 'no-store') |
| |
| def PolicyAtDate(self, timestamp): |
| """Returns the caching policy at an aribitrary timestamp. |
| |
| Args: |
| timestamp: (float) Seconds since Epoch. |
| |
| Returns: |
| A policy in POLICIES. |
| """ |
| # Note: the implementation is largely transcribed from |
| # net/http/http_response_headers.cc, itself following RFC 2616. |
| if not self.IsCacheable(): |
| return self.FETCH |
| freshness = self.GetFreshnessLifetimes() |
| if freshness[0] == 0 and freshness[1] == 0: |
| return self.VALIDATION_SYNC |
| age = self._GetCurrentAge(timestamp) |
| if freshness[0] > age: |
| return self.VALIDATION_NONE |
| if (freshness[0] + freshness[1]) > age: |
| return self.VALIDATION_ASYNC |
| return self.VALIDATION_SYNC |
| |
| def GetFreshnessLifetimes(self): |
| """Returns [freshness, stale-while-revalidate freshness] in seconds.""" |
| # This is adapted from GetFreshnessLifetimes() in |
| # //net/http/http_response_headers.cc (which follows the RFC). |
| r = self.request |
| result = [0, 0] |
| if (r.HasResponseHeaderValue('Cache-Control', 'no-cache') |
| or r.HasResponseHeaderValue('Cache-Control', 'no-store') |
| or r.HasResponseHeaderValue('Vary', '*')): # RFC 2616, 13.6. |
| return result |
| must_revalidate = r.HasResponseHeaderValue( |
| 'Cache-Control', 'must-revalidate') |
| swr_header = r.GetCacheControlDirective('stale-while-revalidate') |
| if not must_revalidate and swr_header: |
| result[1] = _ParseStringToInt(swr_header) |
| |
| max_age_header = r.GetCacheControlDirective('max-age') |
| if max_age_header: |
| result[0] = _ParseStringToInt(max_age_header) |
| return result |
| |
| date = self._GetDateValue('Date') or self._response_time |
| expires = self._GetDateValue('Expires') |
| if expires: |
| result[0] = expires - date |
| return result |
| |
| if self.request.status in (200, 203, 206) and not must_revalidate: |
| last_modified = self._GetDateValue('Last-Modified') |
| if last_modified and last_modified < date: |
| result[0] = (date - last_modified) / 10 |
| return result |
| |
| if self.request.status in (300, 301, 308, 410): |
| return [2**48, 0] # ~forever. |
| # No header -> not fresh. |
| return result |
| |
| def _GetDateValue(self, name): |
| date_str = self.request.GetHTTPResponseHeader(name) |
| if not date_str: |
| return None |
| parsed_date = email.utils.parsedate_tz(date_str) |
| if parsed_date is None: |
| return None |
| return email.utils.mktime_tz(parsed_date) |
| |
| def _GetCurrentAge(self, current_time): |
| # See GetCurrentAge() in //net/http/http_response_headers.cc. |
| r = self.request |
| date_value = self._GetDateValue('Date') or self._response_time |
| age_value = int(r.GetHTTPResponseHeader('Age') or '0') |
| |
| apparent_age = max(0, self._response_time - date_value) |
| corrected_received_age = max(apparent_age, age_value) |
| response_delay = self._response_time - self._request_time |
| corrected_initial_age = corrected_received_age + response_delay |
| resident_time = current_time - self._response_time |
| current_age = corrected_initial_age + resident_time |
| |
| return current_age |
| |
| |
| class RequestTrack(devtools_monitor.Track): |
| """Aggregates request data.""" |
| _REDIRECT_SUFFIX = '.redirect' |
| # Request status |
| _STATUS_SENT = 0 |
| _STATUS_RESPONSE = 1 |
| _STATUS_DATA = 2 |
| _STATUS_FINISHED = 3 |
| _STATUS_FAILED = 4 |
| # Serialization KEYS |
| _EVENTS_KEY = 'events' |
| _METADATA_KEY = 'metadata' |
| _DUPLICATES_KEY = 'duplicates_count' |
| _INCONSISTENT_INITIATORS_KEY = 'inconsistent_initiators' |
| def __init__(self, connection): |
| super(RequestTrack, self).__init__(connection) |
| self._connection = connection |
| self._requests = [] |
| self._requests_in_flight = {} # requestId -> (request, status) |
| self._completed_requests_by_id = {} |
| self._redirects_count_by_id = collections.defaultdict(int) |
| self._indexed = False |
| self._request_start_timestamps = None |
| self._request_end_timestamps = None |
| self._requests_by_start = None |
| self._requests_by_end = None |
| if connection: # Optional for testing. |
| for method in RequestTrack._METHOD_TO_HANDLER: |
| self._connection.RegisterListener(method, self) |
| # Enable asynchronous callstacks to get full javascript callstacks in |
| # initiators |
| self._connection.SetScopedState('Debugger.setAsyncCallStackDepth', |
| {'maxDepth': 4}, {'maxDepth': 0}, True) |
| # responseReceived message are sometimes duplicated. Records the message to |
| # detect this. |
| self._request_id_to_response_received = {} |
| self.duplicates_count = 0 |
| self.inconsistent_initiators_count = 0 |
| |
| def Handle(self, method, msg): |
| assert method in RequestTrack._METHOD_TO_HANDLER |
| self._indexed = False |
| params = msg['params'] |
| request_id = params['requestId'] |
| RequestTrack._METHOD_TO_HANDLER[method](self, request_id, params) |
| |
| def GetEvents(self): |
| if self._requests_in_flight: |
| logging.warning('Number of requests still in flight: %d.' |
| % len(self._requests_in_flight)) |
| return self._requests |
| |
| def GetFirstResourceRequest(self): |
| return self.GetEvents()[0] |
| |
| def GetFirstRequestMillis(self): |
| """Find the canonical start time for this track. |
| |
| Returns: |
| The millisecond timestamp of the first request. |
| """ |
| assert self._requests, "No requests to analyze." |
| self._IndexRequests() |
| return self._request_start_timestamps[0] |
| |
| def GetLastRequestMillis(self): |
| """Find the canonical start time for this track. |
| |
| Returns: |
| The millisecond timestamp of the first request. |
| """ |
| assert self._requests, "No requests to analyze." |
| self._IndexRequests() |
| return self._request_end_timestamps[-1] |
| |
| def GetEventsStartingBetween(self, start_ms, end_ms): |
| """Return events that started in a range. |
| |
| Args: |
| start_ms: the start time to query, in milliseconds from the first request. |
| end_ms: the end time to query, in milliseconds from the first request. |
| |
| Returns: |
| A list of requests whose start time is in [start_ms, end_ms]. |
| """ |
| self._IndexRequests() |
| low = bisect.bisect_left(self._request_start_timestamps, start_ms) |
| high = bisect.bisect_right(self._request_start_timestamps, end_ms) |
| return self._requests_by_start[low:high] |
| |
| def GetEventsEndingBetween(self, start_ms, end_ms): |
| """Return events that ended in a range. |
| |
| Args: |
| start_ms: the start time to query, in milliseconds from the first request. |
| end_ms: the end time to query, in milliseconds from the first request. |
| |
| Returns: |
| A list of requests whose end time is in [start_ms, end_ms]. |
| """ |
| self._IndexRequests() |
| low = bisect.bisect_left(self._request_end_timestamps, start_ms) |
| high = bisect.bisect_right(self._request_end_timestamps, end_ms) |
| return self._requests_by_end[low:high] |
| |
| def ToJsonDict(self): |
| if self._requests_in_flight: |
| logging.warning('Requests in flight, will be ignored in the dump') |
| return {self._EVENTS_KEY: [ |
| request.ToJsonDict() for request in self._requests], |
| self._METADATA_KEY: { |
| self._DUPLICATES_KEY: self.duplicates_count, |
| self._INCONSISTENT_INITIATORS_KEY: |
| self.inconsistent_initiators_count}} |
| |
| @classmethod |
| def FromJsonDict(cls, json_dict): |
| assert cls._EVENTS_KEY in json_dict |
| assert cls._METADATA_KEY in json_dict |
| result = RequestTrack(None) |
| requests = [Request.FromJsonDict(request) |
| for request in json_dict[cls._EVENTS_KEY]] |
| result._requests = requests |
| metadata = json_dict[cls._METADATA_KEY] |
| result.duplicates_count = metadata.get(cls._DUPLICATES_KEY, 0) |
| result.inconsistent_initiators_count = metadata.get( |
| cls._INCONSISTENT_INITIATORS_KEY, 0) |
| return result |
| |
| def _IndexRequests(self): |
| # TODO(mattcary): if we ever have requests without timing then we either |
| # need a default, or to make an index that only includes requests with |
| # timings. |
| if self._indexed: |
| return |
| valid_requests = [r for r in self._requests |
| if r.start_msec is not None] |
| self._requests_by_start = sorted(valid_requests, |
| key=lambda r: r.start_msec) |
| self._request_start_timestamps = [r.start_msec |
| for r in self._requests_by_start] |
| self._requests_by_end = sorted(valid_requests, |
| key=lambda r: r.end_msec) |
| self._request_end_timestamps = [r.end_msec |
| for r in self._requests_by_end] |
| self._indexed = True |
| |
| def _RequestWillBeSent(self, request_id, params): |
| # Several "requestWillBeSent" events can be dispatched in a row in the case |
| # of redirects. |
| redirect_initiator = None |
| if request_id in self._completed_requests_by_id: |
| assert request_id not in self._requests_in_flight |
| return |
| if request_id in self._requests_in_flight: |
| redirect_initiator = self._HandleRedirect(request_id, params) |
| assert (request_id not in self._requests_in_flight) |
| r = Request() |
| r.request_id = request_id |
| _CopyFromDictToObject( |
| params, r, (('frameId', 'frame_id'), ('loaderId', 'loader_id'), |
| ('documentURL', 'document_url'), |
| ('timestamp', 'timestamp'), ('wallTime', 'wall_time'), |
| ('initiator', 'initiator'))) |
| request = params['request'] |
| _CopyFromDictToObject( |
| request, r, (('url', 'url'), ('method', 'method'), |
| ('headers', 'headers'), |
| ('initialPriority', 'initial_priority'))) |
| r.resource_type = params.get('type', 'Other') |
| if redirect_initiator: |
| original_initiator = r.initiator |
| r.initiator = redirect_initiator |
| r.initiator[Request.ORIGINAL_INITIATOR] = original_initiator |
| initiating_request = self._completed_requests_by_id[ |
| redirect_initiator[Request.INITIATING_REQUEST]] |
| initiating_initiator = initiating_request.initiator.get( |
| Request.ORIGINAL_INITIATOR, initiating_request.initiator) |
| if initiating_initiator != original_initiator: |
| self.inconsistent_initiators_count += 1 |
| self._requests_in_flight[request_id] = (r, RequestTrack._STATUS_SENT) |
| |
| def _HandleRedirect(self, request_id, params): |
| (r, status) = self._requests_in_flight[request_id] |
| assert status == RequestTrack._STATUS_SENT |
| # The second request contains timing information pertaining to the first |
| # one. Finalize the first request. |
| assert 'redirectResponse' in params |
| redirect_response = params['redirectResponse'] |
| |
| _CopyFromDictToObject(redirect_response, r, |
| (('headers', 'response_headers'), |
| ('encodedDataLength', 'response_headers_length'), |
| ('fromDiskCache', 'from_disk_cache'), |
| ('protocol', 'protocol'), ('status', 'status'), |
| ('statusText', 'status_text'))) |
| r.timing = Timing.FromDevToolsDict(redirect_response['timing']) |
| |
| redirect_index = self._redirects_count_by_id[request_id] |
| self._redirects_count_by_id[request_id] += 1 |
| r.request_id = '%s%s.%d' % (request_id, self._REDIRECT_SUFFIX, |
| redirect_index + 1) |
| initiator = { |
| 'type': 'redirect', Request.INITIATING_REQUEST: r.request_id} |
| self._requests_in_flight[r.request_id] = (r, RequestTrack._STATUS_FINISHED) |
| del self._requests_in_flight[request_id] |
| self._FinalizeRequest(r.request_id) |
| return initiator |
| |
| def _RequestServedFromCache(self, request_id, _): |
| if request_id not in self._requests_in_flight: |
| return |
| (request, status) = self._requests_in_flight[request_id] |
| assert status == RequestTrack._STATUS_SENT |
| request.served_from_cache = True |
| |
| def _ResponseReceived(self, request_id, params): |
| if request_id in self._completed_requests_by_id: |
| assert request_id not in self._requests_in_flight |
| return |
| assert request_id in self._requests_in_flight |
| (r, status) = self._requests_in_flight[request_id] |
| if status == RequestTrack._STATUS_RESPONSE: |
| # Duplicated messages (apart from the timestamp) are OK. |
| old_params = self._request_id_to_response_received[request_id] |
| params_copy = copy.deepcopy(params) |
| params_copy['timestamp'] = None |
| old_params['timestamp'] = None |
| assert params_copy == old_params |
| self.duplicates_count += 1 |
| return |
| assert status == RequestTrack._STATUS_SENT |
| assert (r.frame_id == params['frameId'] or |
| params['response']['protocol'] == 'data') |
| assert r.timestamp <= params['timestamp'] |
| if r.resource_type == 'Other': |
| r.resource_type = params.get('type', 'Other') |
| else: |
| assert r.resource_type == params.get('type', 'Other') |
| response = params['response'] |
| _CopyFromDictToObject( |
| response, r, (('status', 'status'), ('mimeType', 'mime_type'), |
| ('fromDiskCache', 'from_disk_cache'), |
| ('fromServiceWorker', 'from_service_worker'), |
| ('protocol', 'protocol'), ('statusText', 'status_text'), |
| # Actual request headers are not known before reaching the |
| # network stack. |
| ('requestHeaders', 'request_headers'), |
| ('encodedDataLength', 'response_headers_length'), |
| ('headers', 'response_headers'))) |
| timing_dict = {} |
| # Some URLs don't have a timing dict (e.g. data URLs), and timings for |
| # cached requests are stale. |
| # TODO(droger): the timestamp is inacurate, get the real timings instead. |
| if not response.get('timing') or r.served_from_cache: |
| timing_dict = {'requestTime': r.timestamp} |
| else: |
| timing_dict = response['timing'] |
| r.timing = Timing.FromDevToolsDict(timing_dict) |
| self._requests_in_flight[request_id] = (r, RequestTrack._STATUS_RESPONSE) |
| self._request_id_to_response_received[request_id] = params |
| |
| def _DataReceived(self, request_id, params): |
| if request_id not in self._requests_in_flight: |
| return |
| (r, status) = self._requests_in_flight[request_id] |
| assert (status == RequestTrack._STATUS_RESPONSE |
| or status == RequestTrack._STATUS_DATA) |
| offset = r._TimestampOffsetFromStartMs(params['timestamp']) |
| r.data_chunks.append((offset, params['encodedDataLength'])) |
| self._requests_in_flight[request_id] = (r, RequestTrack._STATUS_DATA) |
| |
| def _LoadingFinished(self, request_id, params): |
| if request_id not in self._requests_in_flight: |
| return |
| (r, status) = self._requests_in_flight[request_id] |
| assert (status == RequestTrack._STATUS_RESPONSE |
| or status == RequestTrack._STATUS_DATA) |
| r.encoded_data_length = params['encodedDataLength'] |
| r.timing.loading_finished = r._TimestampOffsetFromStartMs( |
| params['timestamp']) |
| self._requests_in_flight[request_id] = (r, RequestTrack._STATUS_FINISHED) |
| self._FinalizeRequest(request_id) |
| |
| def _LoadingFailed(self, request_id, params): |
| if request_id not in self._requests_in_flight: |
| logging.warning('An unknown request failed: %s' % request_id) |
| return |
| (r, _) = self._requests_in_flight[request_id] |
| r.failed = True |
| r.error_text = params['errorText'] |
| self._requests_in_flight[request_id] = (r, RequestTrack._STATUS_FINISHED) |
| self._FinalizeRequest(request_id) |
| |
| def _FinalizeRequest(self, request_id): |
| (request, status) = self._requests_in_flight[request_id] |
| assert status == RequestTrack._STATUS_FINISHED |
| del self._requests_in_flight[request_id] |
| self._completed_requests_by_id[request_id] = request |
| self._requests.append(request) |
| |
| def __eq__(self, o): |
| return self._requests == o._requests |
| |
| |
| RequestTrack._METHOD_TO_HANDLER = { |
| 'Network.requestWillBeSent': RequestTrack._RequestWillBeSent, |
| 'Network.requestServedFromCache': RequestTrack._RequestServedFromCache, |
| 'Network.responseReceived': RequestTrack._ResponseReceived, |
| 'Network.dataReceived': RequestTrack._DataReceived, |
| 'Network.loadingFinished': RequestTrack._LoadingFinished, |
| 'Network.loadingFailed': RequestTrack._LoadingFailed} |
| |
| |
| def _CopyFromDictToObject(d, o, key_attrs): |
| for (key, attr) in key_attrs: |
| if key in d: |
| setattr(o, attr, d[key]) |
| |
| |
| if __name__ == '__main__': |
| import json |
| import sys |
| events = json.load(open(sys.argv[1], 'r')) |
| request_track = RequestTrack(None) |
| for event in events: |
| event_method = event['method'] |
| request_track.Handle(event_method, event) |
