| #!/usr/bin/env python |
| # |
| # Copyright 2007 Google Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| |
| |
| |
| """Main script for appstats analytics.""" |
| |
| |
| |
| import email.Utils |
| try: |
| import json |
| except ImportError: |
| import simplejson as json |
| import logging |
| import mimetypes |
| import os |
| import re |
| import time |
| from google.appengine.ext import webapp |
| from google.appengine.ext.analytics import process |
| from google.appengine.ext.analytics import stats |
| from google.appengine.ext.appstats import loader |
| from google.appengine.ext.appstats import recording |
| from google.appengine.ext.appstats import ui |
| from google.appengine.ext.webapp import template |
| from google.appengine.ext.webapp import util |
| |
| |
| class Cache(object): |
| """Cache appstats records for better tool performance. |
| |
| Loading full Appstats records from file is time |
| consuming mainly because of the overheads in converting binary |
| protobuf strings to python protobuf objects. Caching the |
| records can help ensure this is done only the first time the |
| main page is loaded, and the overheads are avoided as the user |
| navigates the tool. Note that caching is intended for the offline |
| analysis case (e.g. analyzing locally downloaded files). In online |
| production, local caches might not be effective as requests go to |
| multiple app servers. Also, there might be issues around memcache |
| getting updated periodically. Note that we store the file name |
| and the time the file has been last modified to identify if the |
| cache is still valid. |
| """ |
| |
| def __init__(self): |
| """Constructor.""" |
| self.hascontents = False |
| self.filename = None |
| |
| self.mtime = None |
| self.recordlist = [] |
| |
| def Reset(self): |
| """Reset and delete cache contents.""" |
| self.hascontents = False |
| self.filename = None |
| self.mtime = None |
| self.recordlist = [] |
| |
| def IsCached(self, source, mtime): |
| """Check whether data from a file is cached. |
| |
| Args: |
| source: name of file being read |
| mtime: last modification time of file being read |
| |
| Returns: |
| A boolean: true if cached, false otherwise. |
| """ |
| if not self.hascontents: |
| return False |
| if self.filename == source and self.mtime == mtime: |
| return True |
| else: |
| return False |
| |
| def Insert(self, source, mtime, recordlist): |
| """Insert records in cache. |
| |
| Args: |
| source: name of file whose data is being cached. |
| mtime: last modification time of file being cached. |
| recordlist: list of StatsProto instances retrieved from |
| file in reverse chronological order (i.e. most recent first). |
| """ |
| self.hascontents = True |
| self.filename = source |
| self.mtime = mtime |
| self.recordlist = recordlist |
| |
| |
| class Filter(object): |
| """Returns a subset of records that meet filtering crtieria. |
| |
| While navigating the tool, developers may wish to focus on a certain |
| subset of records that meet desired filters. This class is used to |
| specify the desired filtering criteria. Currently, the supported filters |
| are (i) by time of recording; and (ii) request latency. |
| """ |
| |
| url = None |
| starttime = None |
| endtime = None |
| latency_lower = None |
| latency_upper = None |
| |
| def __init__(self, url=None, starttime=None, endtime=None, |
| latency_lower=None, latency_upper=None): |
| """Set filtering criteria. |
| |
| Args: |
| url: consider only requests corresponding to this URL. |
| starttime: consider only records recorded with timestamp (in seconds) |
| higher than this value. Timestamps are measured from start of recording |
| of entire data source. |
| endtime: consider only records recorded with timestamp (in seconds) |
| lower than this value. |
| latency_lower: consider only requests with latency (in milliseconds) |
| greater than this value |
| latency_upper: consider only requests with latency lower than this value |
| """ |
| self.url = url |
| if starttime: |
| self.starttime = int(starttime) |
| if endtime: |
| self.endtime = int(endtime) |
| if latency_lower: |
| self.latency_lower = int(latency_lower) |
| if latency_upper: |
| self.latency_upper = int(latency_upper) |
| logging.info('Filtering requests: url: %s start: %s end: %s' |
| 'latency_lower: %s, latency_upper: %s', |
| url, starttime, endtime, latency_lower, latency_upper) |
| |
| def Match(self, url, timestamp, latency): |
| """Check if record meets filtering criteria. |
| |
| Args: |
| url: path of that http request (after normalization) |
| timestamp: timestamp of record |
| latency: latency of request that record pertains to. |
| |
| Returns: |
| Boolean which is True if the record matches filtering criteria |
| and false otherwise. |
| """ |
| if self.url: |
| if url != self.url: |
| return False |
| if self.starttime: |
| if timestamp < self.starttime: |
| return False |
| if self.endtime: |
| if timestamp > self.endtime: |
| return False |
| if self.latency_lower: |
| if latency < self.latency_lower: |
| return False |
| if self.latency_upper: |
| if latency > self.latency_upper: |
| return False |
| return True |
| |
| |
| cache = Cache() |
| mockable_open = open |
| |
| |
| class StatsPage(webapp.RequestHandler): |
| """Handler for analysis page.""" |
| |
| dirname = os.path.join(os.path.dirname(__file__)) |
| |
| def FilterRecords(self, recordlist, recording_starttime): |
| """Returns subset of records that meet filtering crtieria. |
| |
| While navigating the tool, developers may wish to focus on a certain |
| subset of records that meet desired filters. Currently, the supported |
| filters are (i) by time of recording; and (ii) request latency. Filter |
| information is parsed from request arguments. |
| |
| Args: |
| recordlist: List of raw appstats records over which filtering condition |
| must be applied. |
| recording_starttime: Timestamp when recording of data starts expressed |
| in seconds. This is the timestamp of the earliest recorded Appstats |
| data. |
| Returns: |
| Subset of records that meet the filtering criteria |
| """ |
| url = self.request.get('url') |
| latency_lower = self.request.get('latency_lower') |
| latency_upper = self.request.get('latency_upper') |
| starttime = self.request.get('starttime') |
| endtime = self.request.get('endtime') |
| filter_condition = Filter(url=url, |
| starttime=starttime, |
| endtime=endtime, |
| latency_lower=latency_lower, |
| latency_upper=latency_upper) |
| filtered_records = [] |
| for record in recordlist: |
| path_key = recording.config.extract_key(record) |
| |
| |
| reltime = int(record.start_timestamp_milliseconds() * 0.001 - |
| recording_starttime) |
| latency = record.duration_milliseconds() |
| ismatch = filter_condition.Match(path_key, reltime, latency) |
| if ismatch: |
| filtered_records.append(record) |
| logging.info('Original number of records: %d', len(recordlist)) |
| logging.info('After filtering: number of records: %d', |
| len(filtered_records)) |
| return filter_condition, filtered_records |
| |
| def RenderMain(self, urlstatsdict, source, recording_starttime): |
| """Rendering main page of analysis page. |
| |
| Args: |
| urlstatsdict: A dictionary with keys being URL paths, and values |
| being URLStat objects. |
| source: Source of Appstats data. Either filename if being read from |
| a file or MEMCACHE if being read from memcache. |
| recording_starttime: Timestamp when recording of data starts expressed |
| in seconds. This is the timestamp of the earliest recorded Appstats |
| data. |
| """ |
| resptime_byfreq, intervals = process.URLFreqRespTime(urlstatsdict) |
| data = { |
| 'resptime_byfreq': resptime_byfreq, |
| 'intervals': intervals, |
| 'source': source, |
| 'recordingstart': time.asctime(time.gmtime(recording_starttime)), |
| } |
| path = os.path.join(self.dirname, 'templates/main.html') |
| self.response.out.write(template.render(path, data)) |
| |
| def RenderDrill(self, url, urlstatsdict, recording_starttime, source, |
| filter_condition): |
| """Rendering analysis page that drills into URL. |
| |
| Args: |
| url: URL that is being drilled into. |
| urlstatsdict: A dictionary with keys being URL paths, and values |
| being URLStat objects. |
| recording_starttime: Timestamp when recording of data starts expressed |
| in seconds. This is the timestamp of the earliest recorded Appstats |
| data. |
| source: Source of Appstats data. Either filename if being read from |
| a file or MEMCACHE if being read from memcache. |
| filter_condition: Filter object that specifies filtering criteria on |
| which requests must be shown. |
| """ |
| if url in urlstatsdict: |
| urlstats = urlstatsdict[url] |
| drill = process.DrillURL(urlstats) |
| data = { |
| 'url': url, |
| 'drill': drill, |
| 'first_timestamp': recording_starttime, |
| 'recordingstart': time.asctime(time.gmtime(recording_starttime)), |
| 'source': source, |
| 'filter_json': json.dumps(filter_condition.__dict__), |
| 'filter': filter_condition.__dict__, |
| } |
| path = os.path.join(self.dirname, 'templates/drill.html') |
| self.response.out.write(template.render(path, data)) |
| |
| def RenderDetail(self, url, urlstatsdict, records_bytimestamp, detail): |
| """Renders detailed Appstats view of single request. |
| |
| Args: |
| url: URL that is being drilled into. |
| urlstatsdict: A dictionary with keys being URL paths, and values |
| being URLStat objects. |
| records_bytimestamp: A dictionary. Each key is the timestamp of an |
| Appstats record (expressed in seconds). Each value is the |
| corresponding Appstats record (RequestStatProto protobuf). |
| detail: An index that can help identify which record is being |
| desired. |
| """ |
| if url in urlstatsdict: |
| urlstats = urlstatsdict[url] |
| |
| revindex = -detail - 1 |
| ts = urlstats.urlrequestlist[revindex].timestamp |
| record = records_bytimestamp[ts] |
| ui.render_record(self.response, record) |
| |
| def RenderError(self, errormessage, source): |
| """Render error message page. |
| |
| Args: |
| errormessage: Error message to be rendered. |
| source: Source of Appstats data. Either filename if being read from |
| a file or MEMCACHE if being read from memcache. |
| """ |
| data = { |
| 'errormessage': errormessage, |
| 'source': source, |
| } |
| path = os.path.join(self.dirname, 'templates/error.html') |
| self.response.out.write(template.render(path, data)) |
| |
| def RenderPklList(self, pklfiles): |
| """Render directory listing of all pkl files. |
| |
| Args: |
| pklfiles: A list of pklfiles in the application root directory. |
| """ |
| data = { |
| 'pklfiles': pklfiles, |
| } |
| path = os.path.join(self.dirname, 'templates/showPklFiles.html') |
| self.response.out.write(template.render(path, data)) |
| |
| def ReadableTime(self, seconds): |
| """Convert seconds into user-friendly time. |
| |
| The seconds elapsed since an appstats file is shown on the directory |
| page. This is converted into the most appropriate combination of units |
| (minute, hour or day) to make it easy for the user to read. |
| |
| Args: |
| seconds: Seconds elapsed since an Appstats data file was downloaded. |
| Returns: |
| elapsed: Readable version of seconds elapsed. |
| """ |
| if seconds < 60: |
| if int(seconds) == 1: |
| elapsed = '%d second ago' %(seconds) |
| else: |
| elapsed = '%d seconds ago' %(seconds) |
| elif seconds < 3600: |
| minutes = seconds/60 |
| if int(minutes) == 1: |
| elapsed = '%d minute ago' %(minutes) |
| else: |
| elapsed = '%d minutes ago' %(minutes) |
| elif seconds < 86400: |
| hours = seconds/3600 |
| if int(hours) == 1: |
| elapsed = '%d hour ago' %(hours) |
| else: |
| elapsed = '%d hours ago' %(hours) |
| else: |
| days = seconds/86400 |
| if int(days) == 1: |
| elapsed = '%d day ago' %(days) |
| else: |
| elapsed = '%d days ago' %(days) |
| return elapsed |
| |
| def ListPklFiles(self): |
| """Create a list of available pkl files. |
| |
| Generates a directory listing of application root directory to obtain |
| a list of all pkl files. |
| |
| Returns: |
| pklfiles: A list of tuples one per pkl file in the application |
| root directory. Each tuple contains the file name, seconds |
| elapsed since last modification, and a user-friendly version of elapsed |
| second information. The list is sorted by seconds elapsed, i.e. most |
| recently downloaded files are listed first. |
| """ |
| rootdir = self.GetRoot() |
| files = os.listdir(rootdir) |
| currtime = time.time() |
| pklfiles = [] |
| for filename in files: |
| if re.search('\.pkl$', filename): |
| path = os.path.join(rootdir, filename) |
| lastmodified = os.path.getmtime(path) |
| elapsed_secs = currtime - lastmodified |
| elapsed_text = self.ReadableTime(elapsed_secs) |
| pklfiles.append((filename, elapsed_secs, elapsed_text)) |
| pklfiles.sort(key=lambda tuple: tuple[1]) |
| return pklfiles |
| |
| def GetRoot(self): |
| """Determine the root directory of the application. |
| |
| Returns: |
| Root directory of the application, i.e. directory that has app.yaml |
| file. Returns None if it cannot locate the root directory. |
| """ |
| rootdir = self.dirname |
| tryfile = None |
| while rootdir != '/': |
| tryfile = os.path.join(rootdir, 'app.yaml') |
| if not os.path.exists(tryfile): |
| rootdir = os.path.dirname(rootdir) |
| else: |
| break |
| if rootdir != '/': |
| logging.info('Application Root directory: %s', rootdir) |
| return rootdir |
| else: |
| if os.path.exists('/app.yaml'): |
| logging.info('Application Root directory: %s', rootdir) |
| return rootdir |
| else: |
| logging.error('No parent directory has app.yaml!') |
| return None |
| |
| def ReadData(self, source): |
| """Parses source option and reads appropriate data source. |
| |
| Args: |
| source: Source of Appstats data. Either filename if being read from |
| a file or MEMCACHE if being read from memcache. |
| Returns: |
| errormessage: An error message to display to the user if an error occured |
| while reading data, None if no error occured. |
| recordlist: A list of Appstats records in RequestStatProto protobuf format |
| in reverse chronological order (i.e. most recent first). |
| """ |
| errormessage = None |
| recordlist = None |
| if source == 'MEMCACHE': |
| |
| recordlist = loader.FromMemcache() |
| else: |
| rootdir = self.GetRoot() |
| if rootdir is None: |
| errormessage = 'No parent directory has app.yaml!' |
| return errormessage, recordlist |
| source_root = os.path.join(rootdir, source) |
| try: |
| outfile = mockable_open(source_root, 'rb') |
| except IOError: |
| logging.error('Cannot open %s', source) |
| errormessage = 'Unable to open file!' |
| return errormessage, recordlist |
| mtime = os.path.getmtime(source_root) |
| if cache.IsCached(source, mtime): |
| logging.info('Using cached records from %s', source) |
| recordlist = cache.recordlist |
| else: |
| logging.info('Reading fresh records from %s', source) |
| recordlist = loader.UnpickleFromFile(outfile) |
| cache.Insert(source, mtime, recordlist) |
| return errormessage, recordlist |
| |
| def InitURLStats(self, recordlist): |
| """Initialize data structures from appstats data. |
| |
| Args: |
| recordlist: A list of Appstats records in RequestStatProto protobuf |
| format in reverse chronological order (i.e. most recent first). |
| Returns: |
| records_bytimestamp: A dictionary. Each key is the timestamp of an |
| Appstats record (expressed in seconds). Each value is the |
| corresponding Appstats record (RequestStatProto protobuf). |
| urlstatsdict: A dictionary with keys being URL paths, and values |
| being URLStat objects. |
| """ |
| records_bytimestamp = {} |
| urlstatsdict = {} |
| for record in recordlist: |
| ts = record.start_timestamp_milliseconds() * 0.001 |
| records_bytimestamp[ts] = record |
| path_key = recording.config.extract_key(record) |
| if not path_key in urlstatsdict: |
| urlstatsdict[path_key] = stats.URLStats(path_key) |
| urlstatsdict[path_key].AddRequest(record) |
| return records_bytimestamp, urlstatsdict |
| |
| def get(self): |
| """Handler for statistics/diagnostics page.""" |
| logging.info(self.request.path) |
| if not self.request.path.endswith('/'): |
| querystring = self.request.query_string |
| if not querystring: |
| self.redirect(self.request.path + '/') |
| else: |
| self.redirect(self.request.path + '/?' + self.request.query_string) |
| return |
| if not 'source' in self.request.arguments(): |
| pklfiles = self.ListPklFiles() |
| self.RenderPklList(pklfiles) |
| else: |
| source = self.request.get('source') |
| logging.info('Before ReadData') |
| errormessage, recordlist = self.ReadData(source) |
| logging.info('After ReadData') |
| if errormessage: |
| self.RenderError(errormessage, source) |
| return |
| if not recordlist: |
| self.RenderError('No records in this Appstats snapshot.', source) |
| return |
| |
| |
| recording_starttime = recordlist[-1].start_timestamp_milliseconds() |
| recording_starttime *= 0.001 |
| filter_condition, filtered_records = self.FilterRecords( |
| recordlist, recording_starttime) |
| records_bytimestamp, urlstatsdict = self.InitURLStats(filtered_records) |
| url = self.request.get('url') |
| detail = self.request.get('detail') |
| if not url and not detail: |
| self.RenderMain(urlstatsdict, source, recording_starttime) |
| elif not detail: |
| self.RenderDrill(url, urlstatsdict, recording_starttime, |
| source, filter_condition) |
| else: |
| detail = int(detail) |
| self.RenderDetail(url, urlstatsdict, records_bytimestamp, detail) |
| |
| |
| class LocalStaticHandler(webapp.RequestHandler): |
| """Request handler to serve static files. |
| |
| Only files directory in the static subdirectory are rendered this |
| way (no subdirectories). |
| """ |
| |
| def get(self): |
| """Handler for static page.""" |
| here = os.path.dirname(__file__) |
| fn = self.request.path |
| i = fn.rfind('/') |
| fn = fn[i+1:] |
| fn = os.path.join(here, 'static', fn) |
| ctype, _ = mimetypes.guess_type(fn) |
| assert ctype and '/' in ctype, repr(ctype) |
| expiry = 3600 |
| expiration = email.Utils.formatdate(time.time() + expiry, usegmt=True) |
| fp = mockable_open(fn, 'rb') |
| try: |
| self.response.out.write(fp.read()) |
| finally: |
| fp.close() |
| self.response.headers['Content-type'] = ctype |
| self.response.headers['Cache-Control'] = 'public, max-age=expiry' |
| self.response.headers['Expires'] = expiration |
| |
| |
| URLMAP = [ |
| ('/stats/local/.*', LocalStaticHandler), |
| ('/stats/*', StatsPage), |
| ('/stats/file', ui.FileHandler), |
| ('/stats/static/.*', ui.StaticHandler), |
| ] |
| |
| app = webapp.WSGIApplication(URLMAP, debug=True) |
| |
| |
| def main(): |
| util.run_bare_wsgi_app(app) |
| |
| |
| if __name__ == '__main__': |
| main() |
