google/appengine/ext/appstats/ui.py - external/googleappengine/python - Git at Google

 #!/usr/bin/env python
 #
 # Copyright 2007 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #


 """Web-based User Interface for appstats.

 This is a simple set of webapp-based request handlers that display the
 collected statistics and let you drill down on the information in
 various ways.

 Template files are in the templates/ subdirectory.  Static files are
 in the static/ subdirectory.

 The templates are written to work with either Django 0.96 or Django
 1.0, and most likely they also work with Django 1.1.
 """


 import cgi
 import cStringIO
 import email.Utils
 import logging
 import mimetypes
 import os
 import re
 import sys
 import time
 import traceback

 from google.appengine.api import users
 from google.appengine.ext import admin
 from google.appengine.ext import webapp
 from google.appengine.ext.appstats import datamodel_pb
 from google.appengine.ext.appstats import recording
 from google.appengine.ext.webapp import _template
 from google.appengine.ext.webapp import util

 DEBUG = recording.config.DEBUG


 def _add_billed_ops_to_map(billed_ops_map, billed_ops_list):
   """Adds the BilledOpProto objects to the given map.

   The map is from BilledOpProto.op to the pb.

   Args:
     billed_ops_map: The map to populate.
     billed_ops_list: List containing the BilledOpProtos to add to the map.
   """
   for billed_op in billed_ops_list:
     if billed_op.op() not in billed_ops_map:
       update_me = datamodel_pb.BilledOpProto()
       update_me.set_op(billed_op.op())
       update_me.set_num_ops(0)
       billed_ops_map[billed_op.op()] = update_me
     update_me = billed_ops_map[billed_op.op()]
     update_me.set_num_ops(update_me.num_ops() + billed_op.num_ops())


 def _billed_ops_to_str(billed_ops):
   """Builds a string representation of a list of BilledOpProto."""
   ops_as_strs = []
   for op in billed_ops:
     op_name = datamodel_pb.BilledOpProto.BilledOp_Name(op.op())
     ops_as_strs.append('%s:%s' % (op_name, op.num_ops()))
   return ', '.join(ops_as_strs)


 def _as_percentage_of(cost_micropennies, total_cost_micropennies):
   """The cost as a percentage of the total cost, rounded to hundredths."""
   if total_cost_micropennies == 0:
     return 0
   return round((float(cost_micropennies) / float(total_cost_micropennies))
                * 100, 1)


 def render(tmplname, data):
   """Helper function to render a template."""
   here = os.path.dirname(__file__)
   tmpl = os.path.join(here, 'templates', tmplname)
   data['env'] = os.environ
   data['shell_ok'] = recording.config.SHELL_OK
   data['multithread'] = os.getenv('wsgi.multithread')
   try:
     return _template.render(tmpl, data)
   except Exception, err:
     logging.exception('Failed to render %s', tmpl)
     return 'Problematic template %s: %s' % (tmplname, err)


 class AllStatsInfo(object):
   """AllStats data."""

   def __init__(self, calls, cost, billed_ops):
     self.calls = calls
     self.cost = cost
     self.billed_ops = billed_ops


 class PathStatsInfo(object):
   """PathStats data."""

   def __init__(self, cost, billed_ops, num_requests, most_recent_requests):
     self.cost = cost
     self.billed_ops = billed_ops
     self.num_requests = num_requests
     self.most_recent_requests = most_recent_requests


 class PivotInfo(object):
   """Pivot data. The name attribute can be an rpc or a path."""

   def __init__(self, name, calls, cost, billed_ops, cost_pct):
     self.name = name
     self.calls = calls
     self.cost = cost
     self.billed_ops = billed_ops
     self.cost_pct = cost_pct

   def to_list(self):
     """Convert to a list with values in the locations expected by the ui."""
     return [self.name, self.calls, self.cost, self.billed_ops, self.cost_pct]

   @classmethod
   def from_list(cls, values):
     return cls(values[0], values[1], values[2], values[3], values[4])


 class SummaryHandler(webapp.RequestHandler):
   """Request handler for the main stats page (/stats/)."""

   def get(self):
     recording.dont_record()


     if not self.request.path.endswith('/'):
       self.redirect(self.request.path + '/')
       return


     summaries = recording.load_summary_protos()

     data = self._get_summary_data(summaries)


     self.response.out.write(render('main.html', data))

   def _get_summary_data(self, summaries):
     """Extract statistics from summaries."""
     allstats = {}
     pathstats = {}


     pivot_path_rpc = {}


     pivot_rpc_path = {}

     total_cost_micropennies = 0

     summaries = sorted(summaries,
                        key=lambda x: (-x.start_timestamp_milliseconds()))
     for index, summary in enumerate(summaries):


       path_key = recording.config.extract_key(summary)
       if path_key not in pathstats:
         pathstats[path_key] = PathStatsInfo(0, {}, 1, [index+1])
       else:
         pathstats_info = pathstats[path_key]
         pathstats_info.num_requests += 1

         if len(pathstats_info.most_recent_requests) > 10:
           if pathstats_info.most_recent_requests[-1]:

             pathstats_info.most_recent_requests.append(0)
         else:
           pathstats_info.most_recent_requests.append(index+1)
       if path_key not in pivot_path_rpc:
         pivot_path_rpc[path_key] = {}

       for x in summary.rpc_stats_list():
         rpc_key = x.service_call_name()
         total_calls = x.total_amount_of_calls()


         cost_micropennies = x.total_cost_of_calls_microdollars()
         total_cost_micropennies += cost_micropennies
         pathstats[path_key].cost += cost_micropennies
         _add_billed_ops_to_map(pathstats[path_key].billed_ops,
                                x.total_billed_ops_list())
         if rpc_key in allstats:
           allstats[rpc_key].calls += total_calls
           allstats[rpc_key].cost += cost_micropennies
         else:
           allstats[rpc_key] = AllStatsInfo(total_calls, cost_micropennies, {})
         _add_billed_ops_to_map(
             allstats[rpc_key].billed_ops, x.total_billed_ops_list())
         if rpc_key not in pivot_path_rpc[path_key]:
           pivot_path_rpc[path_key][rpc_key] = PivotInfo(rpc_key, 0, 0, {}, 0)
         pivot_path_rpc[path_key][rpc_key].calls += total_calls
         pivot_path_rpc[path_key][rpc_key].cost += cost_micropennies
         _add_billed_ops_to_map(pivot_path_rpc[path_key][rpc_key].billed_ops,
                                x.total_billed_ops_list())

         if rpc_key not in pivot_rpc_path:
           pivot_rpc_path[rpc_key] = {}
         if path_key not in pivot_rpc_path[rpc_key]:

           pivot_rpc_path[rpc_key][path_key] = PivotInfo(path_key, 0, 0, {}, 0)
         pivot_rpc_path[rpc_key][path_key].calls += total_calls
         pivot_rpc_path[rpc_key][path_key].cost += cost_micropennies
         _add_billed_ops_to_map(pivot_rpc_path[rpc_key][path_key].billed_ops,
                                x.total_billed_ops_list())


     allstats_by_count = []
     for k, v in allstats.iteritems():
       for path_vals in pivot_rpc_path[k].itervalues():
         path_vals.billed_ops = _billed_ops_to_str(
             path_vals.billed_ops.itervalues())
         path_vals.cost_pct = _as_percentage_of(
             path_vals.cost, total_cost_micropennies)


       pivot = sorted(pivot_rpc_path[k].itervalues(),
                      key=lambda x: (-x.calls, x.name))
       allstats_by_count.append((
           k, v.calls, v.cost, _billed_ops_to_str(v.billed_ops.itervalues()),
           _as_percentage_of(v.cost, total_cost_micropennies),
           [x.to_list() for x in pivot]))
     allstats_by_count.sort(key=lambda x: (-x[1], x[0]))


     pathstats_by_count = []
     for path_key, pathstats_info in pathstats.iteritems():
       rpc_count = 0
       for rpc_vals in pivot_path_rpc[path_key].itervalues():
         rpc_vals.billed_ops = _billed_ops_to_str(
             rpc_vals.billed_ops.itervalues())
         rpc_vals.cost_pct = _as_percentage_of(
             rpc_vals.cost, total_cost_micropennies)
         rpc_count += rpc_vals.calls


       pivot = sorted(pivot_path_rpc[path_key].itervalues(),
                      key=lambda x: (-x.calls, x.name))
       pathstats_by_count.append((
           path_key, rpc_count, pathstats_info.cost,
           _billed_ops_to_str(pathstats_info.billed_ops.itervalues()),
           _as_percentage_of(pathstats_info.cost, total_cost_micropennies),
           pathstats_info.num_requests,
           pathstats_info.most_recent_requests,
           [x.to_list() for x in pivot]))

     pathstats_by_count.sort(key=lambda x: (-x[1], -x[5], x[0]))


     return {'requests': summaries,
             'allstats_by_count': allstats_by_count,
             'pathstats_by_count': pathstats_by_count,
             }


 class DetailsHandler(webapp.RequestHandler):
   """Request handler for the details page (/stats/details)."""

   def get(self):
     recording.dont_record()


     time_key = self.request.get('time')
     timestamp = None
     record = None
     if time_key:
       try:
         timestamp = int(time_key) * 0.001
       except Exception:
         pass
     if timestamp:
       record = recording.load_full_proto(timestamp)
     render_record(self.response, record, './file')


 def render_record(response, record, file_url=None, extra_data=None):
   """Render an appstats record in detail.

   This is a minor refactoring of DetailsHandler to support an offline
   tool for analyzing Appstats data and to allow that tool to call
   the original Appstats detailed record visualization. Since the offline
   tool may read Appstats records from other sources (e.g., a downloaded file),
   we are moving the logic of DetailsHandler related to processing and
   visualizing individual Appstats records to this function. This
   function may now be called from outside this file.

   Args:
     response: An instance of the webapp response class representing
       data to be sent in response to a web request.
     record: A RequestStatProto which contains detailed Appstats recording
       for an individual request.
     file_url: Indicates the URL to be used to follow links to files in
       application source code. A default value of 'None' indicates that
       links to files in source code will not be shown.
     extra_data: Optional dict of additional parameters for template.
   """

   data = {}
   if extra_data is not None:
     data.update(extra_data)


   if record is None:

     if extra_data is None:
       response.set_status(404)

     response.out.write(render('details.html', data))
     return

   data.update(get_details_data(record, file_url))
   response.out.write(render('details.html', data))


 def get_details_data(record, file_url=None):
   """ Calculate detailed appstats data for a single request.

   Args:
     record: A RequestStatProto which contains detailed Appstats recording
       for an individual request.
     file_url: Indicates the URL to be used to follow links to files in
       application source code. A default value of 'None' indicates that
       links to files in source code will not be shown.

   Returns:
     A dictionary containing detailed appstats data for a single request.
   """

   rpcstats_map = {}
   for rpc_stat in record.individual_stats_list():
     key = rpc_stat.service_call_name()
     count, real, api, rpc_cost_micropennies, billed_ops = rpcstats_map.get(
         key, (0, 0, 0, 0, {}))
     count += 1
     real += rpc_stat.duration_milliseconds()
     api += rpc_stat.api_mcycles()


     rpc_cost_micropennies += rpc_stat.call_cost_microdollars()
     _add_billed_ops_to_map(billed_ops, rpc_stat.billed_ops_list())
     rpcstats_map[key] = (count, real, api, rpc_cost_micropennies, billed_ops)
   rpcstats_by_count = [
       (name, count, real, recording.mcycles_to_msecs(api),
        rpc_cost_micropennies, _billed_ops_to_str(billed_ops.itervalues()))
       for name, (count, real, api, rpc_cost_micropennies, billed_ops)
       in rpcstats_map.iteritems()]
   rpcstats_by_count.sort(key=lambda x: -x[1])


   real_total = 0
   api_total_mcycles = 0
   for i, rpc_stat in enumerate(record.individual_stats_list()):
     real_total += rpc_stat.duration_milliseconds()
     api_total_mcycles += rpc_stat.api_mcycles()

   api_total = recording.mcycles_to_msecs(api_total_mcycles)

   return {'sys': sys,
           'record': record,
           'rpcstats_by_count': rpcstats_by_count,
           'real_total': real_total,
           'api_total': api_total,
           'file_url': file_url,
           }


 class ShellHandler(webapp.RequestHandler):
   """Request handler for interactive shell.

   This is like /_ah/admin/interactive, but with Appstats output integrated.

   GET displays a form; POST runs some code and displays its output + stats.
   """

   def _check_access(self):
     if recording.config.SHELL_OK:
       return True
     self.response.set_status(403)
     self.response.out.write('You must enable this feature by setting '
                             'appstats_SHELL_OK = True in appengine_config.py')
     return False

   def get(self):
     recording.dont_record()
     if not self._check_access():
       return
     script = self.request.get('script', recording.config.DEFAULT_SCRIPT)
     extra_data = {'is_shell': True,
                   'script': script,
                   'xsrf_token': admin.get_xsrf_token(),
                   }
     render_record(self.response, None, './file', extra_data)

   @admin.xsrf_required
   def post(self):
     recording.dont_record()
     if not self._check_access():
       return

     recorder = recording.Recorder(os.environ)
     recording.recorder_proxy.set_for_current_request(recorder)

     script = self.request.get('script', '').replace('\r\n', '\n')
     output, errors = self.execute_script(script)

     recording.recorder_proxy.clear_for_current_request()
     recorder.record_http_status(0)
     recorder.save()
     record = recorder.get_full_proto()

     extra_data = {'is_shell': True,
                   'script': script,
                   'output': output,
                   'errors': errors,
                   'time_key': int(recorder.start_timestamp * 1000),
                   'xsrf_token': admin.get_xsrf_token(),
                   }
     render_record(self.response, record, './file', extra_data)

   def execute_script(self, script):
     save_stdout = sys.stdout
     save_stderr = sys.stderr
     new_stdout = cStringIO.StringIO()
     new_stderr = cStringIO.StringIO()
     try:
       sys.stdout = new_stdout
       sys.stderr = new_stderr
       exec(script, {})
     except BaseException:
       traceback.print_exc()
     finally:
       sys.stdout = save_stdout
       sys.stderr = save_stderr
       return new_stdout.getvalue(), new_stderr.getvalue()


 class FileHandler(webapp.RequestHandler):
   """Request handler for displaying any text file in the system.

   NOTE: This gives any admin of your app full access to your source code.
   """


   def get(self):
     recording.dont_record()

     lineno = self.request.get('n')
     try:
       lineno = int(lineno)
     except:
       lineno = 0

     filename = self.request.get('f') or ''
     orig_filename = filename

     match = re.match('<path\[(\d+)\]>(.*)', filename)
     if match:
       index, tail = match.groups()
       index = int(index)
       if index < len(sys.path):
         filename = sys.path[index] + tail

     try:
       fp = open(filename)
     except IOError, err:
       self.response.out.write('<h1>IOError</h1><pre>%s</pre>' %
                               cgi.escape(str(err)))
       self.response.set_status(404)
     else:

       try:
         data = {'fp': fp,
                 'filename': filename,
                 'orig_filename': orig_filename,
                 'lineno': lineno,
                 }
         self.response.out.write(render('file.html', data))
       finally:
         fp.close()


 class StaticHandler(webapp.RequestHandler):
   """Request handler to serve static files.

   Only files directory in the static subdirectory are rendered this
   way (no subdirectories).
   """

   def get(self):
     recording.dont_record()
     here = os.path.dirname(__file__)
     fn = self.request.path
     i = fn.rfind('/')
     fn = fn[i+1:]
     fn = os.path.join(here, 'static', fn)
     ctype, encoding = mimetypes.guess_type(fn)
     assert ctype and '/' in ctype, repr(ctype)
     expiry = 3600
     expiration = email.Utils.formatdate(time.time() + expiry, usegmt=True)
     fp = open(fn, 'rb')
     try:
       self.response.out.write(fp.read())
     finally:
       fp.close()
     self.response.headers['Content-type'] = ctype
     self.response.headers['Cache-Control'] = 'public, max-age=expiry'
     self.response.headers['Expires'] = expiration


 URLMAP = [
   ('.*/details', DetailsHandler),
   ('.*/shell', ShellHandler),
   ('.*/file', FileHandler),
   ('.*/static/.*', StaticHandler),
   ('.*', SummaryHandler),
   ]


 class AuthCheckMiddleware(object):
   """Middleware which conducts an auth check."""

   def __init__(self, application):
     self._application = application

   def __call__(self, environ, start_response):
     if not environ.get('SERVER_SOFTWARE', '').startswith('Dev'):
       if not users.is_current_user_admin():
         if users.get_current_user() is None:
           start_response('302 Found',
                          [('Location',
                            users.create_login_url(os.getenv('PATH_INFO', '')))])
           return []
         else:
           start_response('403 Forbidden', [])
           return ['Forbidden\n']
     return self._application(environ, start_response)

 app = AuthCheckMiddleware(webapp.WSGIApplication(URLMAP, debug=DEBUG))


 def main():
   """Main program. Run the auth checking middleware wrapped WSGIApplication."""
   util.run_bare_wsgi_app(app)


 if __name__ == '__main__':
   main()
	#!/usr/bin/env python
	#
	# Copyright 2007 Google Inc.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	#




	"""Web-based User Interface for appstats.

	This is a simple set of webapp-based request handlers that display the
	collected statistics and let you drill down on the information in
	various ways.

	Template files are in the templates/ subdirectory. Static files are
	in the static/ subdirectory.

	The templates are written to work with either Django 0.96 or Django
	1.0, and most likely they also work with Django 1.1.
	"""


	import cgi
	import cStringIO
	import email.Utils
	import logging
	import mimetypes
	import os
	import re
	import sys
	import time
	import traceback

	from google.appengine.api import users
	from google.appengine.ext import admin
	from google.appengine.ext import webapp
	from google.appengine.ext.appstats import datamodel_pb
	from google.appengine.ext.appstats import recording
	from google.appengine.ext.webapp import _template
	from google.appengine.ext.webapp import util

	DEBUG = recording.config.DEBUG


	def _add_billed_ops_to_map(billed_ops_map, billed_ops_list):
	"""Adds the BilledOpProto objects to the given map.

	The map is from BilledOpProto.op to the pb.

	Args:
	billed_ops_map: The map to populate.
	billed_ops_list: List containing the BilledOpProtos to add to the map.
	"""
	for billed_op in billed_ops_list:
	if billed_op.op() not in billed_ops_map:
	update_me = datamodel_pb.BilledOpProto()
	update_me.set_op(billed_op.op())
	update_me.set_num_ops(0)
	billed_ops_map[billed_op.op()] = update_me
	update_me = billed_ops_map[billed_op.op()]
	update_me.set_num_ops(update_me.num_ops() + billed_op.num_ops())


	def _billed_ops_to_str(billed_ops):
	"""Builds a string representation of a list of BilledOpProto."""
	ops_as_strs = []
	for op in billed_ops:
	op_name = datamodel_pb.BilledOpProto.BilledOp_Name(op.op())
	ops_as_strs.append('%s:%s' % (op_name, op.num_ops()))
	return ', '.join(ops_as_strs)


	def _as_percentage_of(cost_micropennies, total_cost_micropennies):
	"""The cost as a percentage of the total cost, rounded to hundredths."""
	if total_cost_micropennies == 0:
	return 0
	return round((float(cost_micropennies) / float(total_cost_micropennies))
	* 100, 1)


	def render(tmplname, data):
	"""Helper function to render a template."""
	here = os.path.dirname(__file__)
	tmpl = os.path.join(here, 'templates', tmplname)
	data['env'] = os.environ
	data['shell_ok'] = recording.config.SHELL_OK
	data['multithread'] = os.getenv('wsgi.multithread')
	try:
	return _template.render(tmpl, data)
	except Exception, err:
	logging.exception('Failed to render %s', tmpl)
	return 'Problematic template %s: %s' % (tmplname, err)


	class AllStatsInfo(object):
	"""AllStats data."""

	def __init__(self, calls, cost, billed_ops):
	self.calls = calls
	self.cost = cost
	self.billed_ops = billed_ops


	class PathStatsInfo(object):
	"""PathStats data."""

	def __init__(self, cost, billed_ops, num_requests, most_recent_requests):
	self.cost = cost
	self.billed_ops = billed_ops
	self.num_requests = num_requests
	self.most_recent_requests = most_recent_requests


	class PivotInfo(object):
	"""Pivot data. The name attribute can be an rpc or a path."""

	def __init__(self, name, calls, cost, billed_ops, cost_pct):
	self.name = name
	self.calls = calls
	self.cost = cost
	self.billed_ops = billed_ops
	self.cost_pct = cost_pct

	def to_list(self):
	"""Convert to a list with values in the locations expected by the ui."""
	return [self.name, self.calls, self.cost, self.billed_ops, self.cost_pct]

	@classmethod
	def from_list(cls, values):
	return cls(values[0], values[1], values[2], values[3], values[4])


	class SummaryHandler(webapp.RequestHandler):
	"""Request handler for the main stats page (/stats/)."""

	def get(self):
	recording.dont_record()





	if not self.request.path.endswith('/'):
	self.redirect(self.request.path + '/')
	return


	summaries = recording.load_summary_protos()

	data = self._get_summary_data(summaries)


	self.response.out.write(render('main.html', data))

	def _get_summary_data(self, summaries):
	"""Extract statistics from summaries."""
	allstats = {}
	pathstats = {}


	pivot_path_rpc = {}


	pivot_rpc_path = {}

	total_cost_micropennies = 0

	summaries = sorted(summaries,
	key=lambda x: (-x.start_timestamp_milliseconds()))
	for index, summary in enumerate(summaries):




	path_key = recording.config.extract_key(summary)
	if path_key not in pathstats:
	pathstats[path_key] = PathStatsInfo(0, {}, 1, [index+1])
	else:
	pathstats_info = pathstats[path_key]
	pathstats_info.num_requests += 1

	if len(pathstats_info.most_recent_requests) > 10:
	if pathstats_info.most_recent_requests[-1]:

	pathstats_info.most_recent_requests.append(0)
	else:
	pathstats_info.most_recent_requests.append(index+1)
	if path_key not in pivot_path_rpc:
	pivot_path_rpc[path_key] = {}

	for x in summary.rpc_stats_list():
	rpc_key = x.service_call_name()
	total_calls = x.total_amount_of_calls()


	cost_micropennies = x.total_cost_of_calls_microdollars()
	total_cost_micropennies += cost_micropennies
	pathstats[path_key].cost += cost_micropennies
	_add_billed_ops_to_map(pathstats[path_key].billed_ops,
	x.total_billed_ops_list())
	if rpc_key in allstats:
	allstats[rpc_key].calls += total_calls
	allstats[rpc_key].cost += cost_micropennies
	else:
	allstats[rpc_key] = AllStatsInfo(total_calls, cost_micropennies, {})
	_add_billed_ops_to_map(
	allstats[rpc_key].billed_ops, x.total_billed_ops_list())
	if rpc_key not in pivot_path_rpc[path_key]:
	pivot_path_rpc[path_key][rpc_key] = PivotInfo(rpc_key, 0, 0, {}, 0)
	pivot_path_rpc[path_key][rpc_key].calls += total_calls
	pivot_path_rpc[path_key][rpc_key].cost += cost_micropennies
	_add_billed_ops_to_map(pivot_path_rpc[path_key][rpc_key].billed_ops,
	x.total_billed_ops_list())

	if rpc_key not in pivot_rpc_path:
	pivot_rpc_path[rpc_key] = {}
	if path_key not in pivot_rpc_path[rpc_key]:

	pivot_rpc_path[rpc_key][path_key] = PivotInfo(path_key, 0, 0, {}, 0)
	pivot_rpc_path[rpc_key][path_key].calls += total_calls
	pivot_rpc_path[rpc_key][path_key].cost += cost_micropennies
	_add_billed_ops_to_map(pivot_rpc_path[rpc_key][path_key].billed_ops,
	x.total_billed_ops_list())





	allstats_by_count = []
	for k, v in allstats.iteritems():
	for path_vals in pivot_rpc_path[k].itervalues():
	path_vals.billed_ops = _billed_ops_to_str(
	path_vals.billed_ops.itervalues())
	path_vals.cost_pct = _as_percentage_of(
	path_vals.cost, total_cost_micropennies)




	pivot = sorted(pivot_rpc_path[k].itervalues(),
	key=lambda x: (-x.calls, x.name))
	allstats_by_count.append((
	k, v.calls, v.cost, _billed_ops_to_str(v.billed_ops.itervalues()),
	_as_percentage_of(v.cost, total_cost_micropennies),
	[x.to_list() for x in pivot]))
	allstats_by_count.sort(key=lambda x: (-x[1], x[0]))


	pathstats_by_count = []
	for path_key, pathstats_info in pathstats.iteritems():
	rpc_count = 0
	for rpc_vals in pivot_path_rpc[path_key].itervalues():
	rpc_vals.billed_ops = _billed_ops_to_str(
	rpc_vals.billed_ops.itervalues())
	rpc_vals.cost_pct = _as_percentage_of(
	rpc_vals.cost, total_cost_micropennies)
	rpc_count += rpc_vals.calls




	pivot = sorted(pivot_path_rpc[path_key].itervalues(),
	key=lambda x: (-x.calls, x.name))
	pathstats_by_count.append((
	path_key, rpc_count, pathstats_info.cost,
	_billed_ops_to_str(pathstats_info.billed_ops.itervalues()),
	_as_percentage_of(pathstats_info.cost, total_cost_micropennies),
	pathstats_info.num_requests,
	pathstats_info.most_recent_requests,
	[x.to_list() for x in pivot]))

	pathstats_by_count.sort(key=lambda x: (-x[1], -x[5], x[0]))


	return {'requests': summaries,
	'allstats_by_count': allstats_by_count,
	'pathstats_by_count': pathstats_by_count,
	}


	class DetailsHandler(webapp.RequestHandler):
	"""Request handler for the details page (/stats/details)."""

	def get(self):
	recording.dont_record()


	time_key = self.request.get('time')
	timestamp = None
	record = None
	if time_key:
	try:
	timestamp = int(time_key) * 0.001
	except Exception:
	pass
	if timestamp:
	record = recording.load_full_proto(timestamp)
	render_record(self.response, record, './file')


	def render_record(response, record, file_url=None, extra_data=None):
	"""Render an appstats record in detail.

	This is a minor refactoring of DetailsHandler to support an offline
	tool for analyzing Appstats data and to allow that tool to call
	the original Appstats detailed record visualization. Since the offline
	tool may read Appstats records from other sources (e.g., a downloaded file),
	we are moving the logic of DetailsHandler related to processing and
	visualizing individual Appstats records to this function. This
	function may now be called from outside this file.

	Args:
	response: An instance of the webapp response class representing
	data to be sent in response to a web request.
	record: A RequestStatProto which contains detailed Appstats recording
	for an individual request.
	file_url: Indicates the URL to be used to follow links to files in
	application source code. A default value of 'None' indicates that
	links to files in source code will not be shown.
	extra_data: Optional dict of additional parameters for template.
	"""

	data = {}
	if extra_data is not None:
	data.update(extra_data)


	if record is None:

	if extra_data is None:
	response.set_status(404)

	response.out.write(render('details.html', data))
	return

	data.update(get_details_data(record, file_url))
	response.out.write(render('details.html', data))


	def get_details_data(record, file_url=None):
	""" Calculate detailed appstats data for a single request.

	Args:
	record: A RequestStatProto which contains detailed Appstats recording
	for an individual request.
	file_url: Indicates the URL to be used to follow links to files in
	application source code. A default value of 'None' indicates that
	links to files in source code will not be shown.

	Returns:
	A dictionary containing detailed appstats data for a single request.
	"""

	rpcstats_map = {}
	for rpc_stat in record.individual_stats_list():
	key = rpc_stat.service_call_name()
	count, real, api, rpc_cost_micropennies, billed_ops = rpcstats_map.get(
	key, (0, 0, 0, 0, {}))
	count += 1
	real += rpc_stat.duration_milliseconds()
	api += rpc_stat.api_mcycles()


	rpc_cost_micropennies += rpc_stat.call_cost_microdollars()
	_add_billed_ops_to_map(billed_ops, rpc_stat.billed_ops_list())
	rpcstats_map[key] = (count, real, api, rpc_cost_micropennies, billed_ops)
	rpcstats_by_count = [
	(name, count, real, recording.mcycles_to_msecs(api),
	rpc_cost_micropennies, _billed_ops_to_str(billed_ops.itervalues()))
	for name, (count, real, api, rpc_cost_micropennies, billed_ops)
	in rpcstats_map.iteritems()]
	rpcstats_by_count.sort(key=lambda x: -x[1])


	real_total = 0
	api_total_mcycles = 0
	for i, rpc_stat in enumerate(record.individual_stats_list()):
	real_total += rpc_stat.duration_milliseconds()
	api_total_mcycles += rpc_stat.api_mcycles()

	api_total = recording.mcycles_to_msecs(api_total_mcycles)

	return {'sys': sys,
	'record': record,
	'rpcstats_by_count': rpcstats_by_count,
	'real_total': real_total,
	'api_total': api_total,
	'file_url': file_url,
	}


	class ShellHandler(webapp.RequestHandler):
	"""Request handler for interactive shell.

	This is like /_ah/admin/interactive, but with Appstats output integrated.

	GET displays a form; POST runs some code and displays its output + stats.
	"""

	def _check_access(self):
	if recording.config.SHELL_OK:
	return True
	self.response.set_status(403)
	self.response.out.write('You must enable this feature by setting '
	'appstats_SHELL_OK = True in appengine_config.py')
	return False

	def get(self):
	recording.dont_record()
	if not self._check_access():
	return
	script = self.request.get('script', recording.config.DEFAULT_SCRIPT)
	extra_data = {'is_shell': True,
	'script': script,
	'xsrf_token': admin.get_xsrf_token(),
	}
	render_record(self.response, None, './file', extra_data)

	@admin.xsrf_required
	def post(self):
	recording.dont_record()
	if not self._check_access():
	return

	recorder = recording.Recorder(os.environ)
	recording.recorder_proxy.set_for_current_request(recorder)

	script = self.request.get('script', '').replace('\r\n', '\n')
	output, errors = self.execute_script(script)

	recording.recorder_proxy.clear_for_current_request()
	recorder.record_http_status(0)
	recorder.save()
	record = recorder.get_full_proto()

	extra_data = {'is_shell': True,
	'script': script,
	'output': output,
	'errors': errors,
	'time_key': int(recorder.start_timestamp * 1000),
	'xsrf_token': admin.get_xsrf_token(),
	}
	render_record(self.response, record, './file', extra_data)

	def execute_script(self, script):
	save_stdout = sys.stdout
	save_stderr = sys.stderr
	new_stdout = cStringIO.StringIO()
	new_stderr = cStringIO.StringIO()
	try:
	sys.stdout = new_stdout
	sys.stderr = new_stderr
	exec(script, {})
	except BaseException:
	traceback.print_exc()
	finally:
	sys.stdout = save_stdout
	sys.stderr = save_stderr
	return new_stdout.getvalue(), new_stderr.getvalue()


	class FileHandler(webapp.RequestHandler):
	"""Request handler for displaying any text file in the system.

	NOTE: This gives any admin of your app full access to your source code.
	"""








	def get(self):
	recording.dont_record()

	lineno = self.request.get('n')
	try:
	lineno = int(lineno)
	except:
	lineno = 0

	filename = self.request.get('f') or ''
	orig_filename = filename

	match = re.match('<path\[(\d+)\]>(.*)', filename)
	if match:
	index, tail = match.groups()
	index = int(index)
	if index < len(sys.path):
	filename = sys.path[index] + tail

	try:
	fp = open(filename)
	except IOError, err:
	self.response.out.write('<h1>IOError</h1><pre>%s</pre>' %
	cgi.escape(str(err)))
	self.response.set_status(404)
	else:

	try:
	data = {'fp': fp,
	'filename': filename,
	'orig_filename': orig_filename,
	'lineno': lineno,
	}
	self.response.out.write(render('file.html', data))
	finally:
	fp.close()


	class StaticHandler(webapp.RequestHandler):
	"""Request handler to serve static files.

	Only files directory in the static subdirectory are rendered this
	way (no subdirectories).
	"""

	def get(self):
	recording.dont_record()
	here = os.path.dirname(__file__)
	fn = self.request.path
	i = fn.rfind('/')
	fn = fn[i+1:]
	fn = os.path.join(here, 'static', fn)
	ctype, encoding = mimetypes.guess_type(fn)
	assert ctype and '/' in ctype, repr(ctype)
	expiry = 3600
	expiration = email.Utils.formatdate(time.time() + expiry, usegmt=True)
	fp = open(fn, 'rb')
	try:
	self.response.out.write(fp.read())
	finally:
	fp.close()
	self.response.headers['Content-type'] = ctype
	self.response.headers['Cache-Control'] = 'public, max-age=expiry'
	self.response.headers['Expires'] = expiration




	URLMAP = [
	('.*/details', DetailsHandler),
	('.*/shell', ShellHandler),
	('.*/file', FileHandler),
	('./static/.', StaticHandler),
	('.*', SummaryHandler),
	]


	class AuthCheckMiddleware(object):
	"""Middleware which conducts an auth check."""

	def __init__(self, application):
	self._application = application

	def __call__(self, environ, start_response):
	if not environ.get('SERVER_SOFTWARE', '').startswith('Dev'):
	if not users.is_current_user_admin():
	if users.get_current_user() is None:
	start_response('302 Found',
	[('Location',
	users.create_login_url(os.getenv('PATH_INFO', '')))])
	return []
	else:
	start_response('403 Forbidden', [])
	return ['Forbidden\n']
	return self._application(environ, start_response)

	app = AuthCheckMiddleware(webapp.WSGIApplication(URLMAP, debug=DEBUG))


	def main():
	"""Main program. Run the auth checking middleware wrapped WSGIApplication."""
	util.run_bare_wsgi_app(app)


	if __name__ == '__main__':
	main()