media/tools/constrained_network_server/cns.py - chromium/src - Git at Google

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 """Constrained Network Server. Serves files with supplied network constraints.

 The CNS exposes a web based API allowing network constraints to be imposed on
 file serving.

 TODO(dalecurtis): Add some more docs here.

 """

 import logging
 import mimetypes
 import optparse
 import os
 import signal
 import sys
 import threading
 import time
 import traffic_control

 try:
   import cherrypy
 except ImportError:
   print ('CNS requires CherryPy v3 or higher to be installed. Please install\n'
          'and try again. On Linux: sudo apt-get install python-cherrypy3\n')
   sys.exit(1)

 # Add webm file types to mimetypes map since cherrypy's default type is text.
 mimetypes.types_map['.webm'] = 'video/webm'

 # Default logging is ERROR. Use --verbose to enable DEBUG logging.
 _DEFAULT_LOG_LEVEL = logging.ERROR

 # Default port to serve the CNS on.
 _DEFAULT_SERVING_PORT = 9000

 # Default port range for constrained use.
 _DEFAULT_CNS_PORT_RANGE = (50000, 51000)

 # Default number of seconds before a port can be torn down.
 _DEFAULT_PORT_EXPIRY_TIME_SECS = 5 * 60


 class PortAllocator(object):
   """Dynamically allocates/deallocates ports with a given set of constraints."""

   def __init__(self, port_range, expiry_time_secs=5 * 60):
     """Sets up initial state for the Port Allocator.

     Args:
       port_range: Range of ports available for allocation.
       expiry_time_secs: Amount of time in seconds before constrained ports are
           cleaned up.
     """
     self._port_range = port_range
     self._expiry_time_secs = expiry_time_secs

     # Keeps track of ports we've used, the creation key, and the last request
     # time for the port so they can be cached and cleaned up later.
     self._ports = {}

     # Locks port creation and cleanup. TODO(dalecurtis): If performance becomes
     # an issue a per-port based lock system can be used instead.
     self._port_lock = threading.Lock()

   def Get(self, key, new_port=False, **kwargs):
     """Sets up a constrained port using the requested parameters.

     Requests for the same key and constraints will result in a cached port being
     returned if possible, subject to new_port.

     Args:
       key: Used to cache ports with the given constraints.
       new_port: Whether to create a new port or use an existing one if possible.
       **kwargs: Constraints to pass into traffic control.

     Returns:
       None if no port can be setup or the port number of the constrained port.
     """
     with self._port_lock:
       # Check port key cache to see if this port is already setup. Update the
       # cache time and return the port if so. Performance isn't a concern here,
       # so just iterate over ports dict for simplicity.
       full_key = (key,) + tuple(kwargs.values())
       if not new_port:
         for port, status in self._ports.iteritems():
           if full_key == status['key']:
             self._ports[port]['last_update'] = time.time()
             return port

       # Cleanup ports on new port requests. Do it after the cache check though
       # so we don't erase and then setup the same port.
       if self._expiry_time_secs > 0:
         self._CleanupLocked(all_ports=False)

       # Performance isn't really an issue here, so just iterate over the port
       # range to find an unused port. If no port is found, None is returned.
       for port in xrange(self._port_range[0], self._port_range[1]):
         if port in self._ports:
           continue
         if self._SetupPort(port, **kwargs):
           kwargs['port'] = port
           self._ports[port] = {'last_update': time.time(), 'key': full_key,
                                'config': kwargs}
           return port

   def _SetupPort(self, port, **kwargs):
     """Setup network constraints on port using the requested parameters.

     Args:
       port: The port number to setup network constraints on.
       **kwargs: Network constraints to set up on the port.

     Returns:
       True if setting the network constraints on the port was successful, false
       otherwise.
     """
     kwargs['port'] = port
     try:
       cherrypy.log('Setting up port %d' % port)
       traffic_control.CreateConstrainedPort(kwargs)
       return True
     except traffic_control.TrafficControlError as e:
       cherrypy.log('Error: %s\nOutput: %s' % (e.msg, e.error))
       return False

   def _CleanupLocked(self, all_ports):
     """Internal cleanup method, expects lock to have already been acquired.

     See Cleanup() for more information.

     Args:
       all_ports: Should all ports be torn down regardless of expiration?
     """
     now = time.time()
     # Use .items() instead of .iteritems() so we can delete keys w/o error.
     for port, status in self._ports.items():
       expired = now - status['last_update'] > self._expiry_time_secs
       if all_ports or expired:
         cherrypy.log('Cleaning up port %d' % port)
         self._DeletePort(port)
         del self._ports[port]

   def _DeletePort(self, port):
     """Deletes network constraints on port.

     Args:
       port: The port number associated with the network constraints.
     """
     try:
       traffic_control.DeleteConstrainedPort(self._ports[port]['config'])
     except traffic_control.TrafficControlError as e:
       cherrypy.log('Error: %s\nOutput: %s' % (e.msg, e.error))

   def Cleanup(self, interface, all_ports=False):
     """Cleans up expired ports, or if all_ports=True, all allocated ports.

     By default, ports which haven't been used for self._expiry_time_secs are
     torn down. If all_ports=True then they are torn down regardless.

     Args:
       interface: Interface the constrained network is setup on.
       all_ports: Should all ports be torn down regardless of expiration?
     """
     with self._port_lock:
       self._CleanupLocked(all_ports)


 class ConstrainedNetworkServer(object):
   """A CherryPy-based HTTP server for serving files with network constraints."""

   def __init__(self, options, port_allocator):
     """Sets up initial state for the CNS.

     Args:
       options: optparse based class returned by ParseArgs()
       port_allocator: A port allocator instance.
     """
     self._options = options
     self._port_allocator = port_allocator

   @cherrypy.expose
   def ServeConstrained(self, f=None, bandwidth=None, latency=None, loss=None,
                        new_port=False):
     """Serves the requested file with the requested constraints.

     Subsequent requests for the same constraints from the same IP will share the
     previously created port unless new_port equals True. If no constraints
     are provided the file is served as is.

     Args:
       f: path relative to http root of file to serve.
       bandwidth: maximum allowed bandwidth for the provided port (integer
           in kbit/s).
       latency: time to add to each packet (integer in ms).
       loss: percentage of packets to drop (integer, 0-100).
       new_port: whether to use a new port for this request or not.
     """
     cherrypy.log('Got request for %s, bandwidth=%s, latency=%s, loss=%s, '
                  'new_port=%s' % (f, bandwidth, latency, loss, new_port))
     # CherryPy is a bit wonky at detecting parameters, so just make them all
     # optional and validate them ourselves.
     if not f:
       raise cherrypy.HTTPError(400, 'Invalid request. File must be specified.')

     # Sanitize and check the path to prevent www-root escapes.
     sanitized_path = os.path.abspath(os.path.join(self._options.www_root, f))
     if not sanitized_path.startswith(self._options.www_root):
       raise cherrypy.HTTPError(403, 'Invalid file requested.')

     # Check existence early to prevent wasted constraint setup.
     if not os.path.exists(sanitized_path):
       raise cherrypy.HTTPError(404, 'File not found.')

     # If there are no constraints, just serve the file.
     if bandwidth is None and latency is None and loss is None:
       return cherrypy.lib.static.serve_file(sanitized_path)

     # Validate inputs. isdigit() guarantees a natural number.
     bandwidth = self._ParseIntParameter(
         bandwidth, 'Invalid bandwidth constraint.', lambda x: x > 0)
     latency = self._ParseIntParameter(
         latency, 'Invalid latency constraint.', lambda x: x >= 0)
     loss = self._ParseIntParameter(
         loss, 'Invalid loss constraint.', lambda x: x <= 100 and x >= 0)

     # Allocate a port using the given constraints. If a port with the requested
     # key is already allocated, it will be reused.
     #
     # TODO(dalecurtis): The key cherrypy.request.remote.ip might not be unique
     # if build slaves are sharing the same VM.
     start_time = time.time()
     constrained_port = self._port_allocator.Get(
         cherrypy.request.remote.ip, server_port=self._options.port,
         interface=self._options.interface, bandwidth=bandwidth, latency=latency,
         loss=loss, new_port=new_port)
     end_time = time.time()

     if not constrained_port:
       raise cherrypy.HTTPError(503, 'Service unavailable. Out of ports.')

     cherrypy.log('Time to set up port %d = %ssec.' %
                  (constrained_port, end_time - start_time))

     # Build constrained URL. Only pass on the file parameter.
     constrained_url = '%s?f=%s' % (
         cherrypy.url().replace(
             ':%d' % self._options.port, ':%d' % constrained_port),
         f)

     # Redirect request to the constrained port.
     cherrypy.lib.cptools.redirect(constrained_url, internal=False)

   def _ParseIntParameter(self, param, msg, check):
     """Returns integer value of param and verifies it satisfies the check.

     Args:
       param: Parameter name to check.
       msg: Message in error if raised.
       check: Check to verify the parameter value.

     Returns:
       None if param is None, integer value of param otherwise.

     Raises:
       cherrypy.HTTPError if param can not be converted to integer or if it does
       not satisfy the check.
     """
     if param:
       try:
         int_value = int(param)
         if check(int_value):
           return int_value
       except:
         pass
       raise cherrypy.HTTPError(400, msg)


 def ParseArgs():
   """Define and parse the command-line arguments."""
   parser = optparse.OptionParser()

   parser.add_option('--expiry-time', type='int',
                     default=_DEFAULT_PORT_EXPIRY_TIME_SECS,
                     help=('Number of seconds before constrained ports expire '
                           'and are cleaned up. 0=Disabled. Default: %default'))
   parser.add_option('--port', type='int', default=_DEFAULT_SERVING_PORT,
                     help='Port to serve the API on. Default: %default')
   parser.add_option('--port-range', default=_DEFAULT_CNS_PORT_RANGE,
                     help=('Range of ports for constrained serving. Specify as '
                           'a comma separated value pair. Default: %default'))
   parser.add_option('--interface', default='eth0',
                     help=('Interface to setup constraints on. Use lo for a '
                           'local client. Default: %default'))
   parser.add_option('--threads', type='int',
                     default=cherrypy._cpserver.Server.thread_pool,
                     help=('Number of threads in the thread pool. Default: '
                           '%default'))
   parser.add_option('--www-root', default=os.getcwd(),
                     help=('Directory root to serve files from. Defaults to the '
                           'current directory: %default'))
   parser.add_option('-v', '--verbose', action='store_true', default=False,
                     help='Turn on verbose output.')

   options = parser.parse_args()[0]

   # Convert port range into the desired tuple format.
   try:
     if isinstance(options.port_range, str):
       options.port_range = [int(port) for port in options.port_range.split(',')]
   except ValueError:
     parser.error('Invalid port range specified.')

   if options.expiry_time < 0:
     parser.error('Invalid expiry time specified.')

   # Convert the path to an absolute to remove any . or ..
   options.www_root = os.path.abspath(options.www_root)

   # Required so that cherrypy logs do not get propagated to root logger causing
   # the logs to be printed twice.
   cherrypy.log.error_log.propagate = False

   _SetLogger(options.verbose)

   return options


 def _SetLogger(verbose):
   # Logging is used for traffic_control debug statements.
   log_level = _DEFAULT_LOG_LEVEL
   if verbose:
     log_level = logging.DEBUG
   logging.basicConfig(level=log_level, format='[%(threadName)s] %(message)s')


 def Main():
   """Configure and start the ConstrainedNetworkServer."""
   options = ParseArgs()

   try:
     traffic_control.CheckRequirements()
   except traffic_control.TrafficControlError as e:
     cherrypy.log(e.msg)
     return

   cherrypy.config.update(
       {'server.socket_host': '::', 'server.socket_port': options.port})

   if options.threads:
     cherrypy.config.update({'server.thread_pool': options.threads})

   # Setup port allocator here so we can call cleanup on failures/exit.
   pa = PortAllocator(options.port_range, expiry_time_secs=options.expiry_time)

   try:
     cherrypy.quickstart(ConstrainedNetworkServer(options, pa))
   finally:
     # Disable Ctrl-C handler to prevent interruption of cleanup.
     signal.signal(signal.SIGINT, lambda signal, frame: None)
     pa.Cleanup(options.interface, all_ports=True)


 if __name__ == '__main__':
   Main()
	#!/usr/bin/env python
	# Copyright (c) 2012 The Chromium Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""Constrained Network Server. Serves files with supplied network constraints.

	The CNS exposes a web based API allowing network constraints to be imposed on
	file serving.

	TODO(dalecurtis): Add some more docs here.

	"""

	import logging
	import mimetypes
	import optparse
	import os
	import signal
	import sys
	import threading
	import time
	import traffic_control

	try:
	import cherrypy
	except ImportError:
	print ('CNS requires CherryPy v3 or higher to be installed. Please install\n'
	'and try again. On Linux: sudo apt-get install python-cherrypy3\n')
	sys.exit(1)

	# Add webm file types to mimetypes map since cherrypy's default type is text.
	mimetypes.types_map['.webm'] = 'video/webm'

	# Default logging is ERROR. Use --verbose to enable DEBUG logging.
	_DEFAULT_LOG_LEVEL = logging.ERROR

	# Default port to serve the CNS on.
	_DEFAULT_SERVING_PORT = 9000

	# Default port range for constrained use.
	_DEFAULT_CNS_PORT_RANGE = (50000, 51000)

	# Default number of seconds before a port can be torn down.
	_DEFAULT_PORT_EXPIRY_TIME_SECS = 5 * 60


	class PortAllocator(object):
	"""Dynamically allocates/deallocates ports with a given set of constraints."""

	def __init__(self, port_range, expiry_time_secs=5 * 60):
	"""Sets up initial state for the Port Allocator.

	Args:
	port_range: Range of ports available for allocation.
	expiry_time_secs: Amount of time in seconds before constrained ports are
	cleaned up.
	"""
	self._port_range = port_range
	self._expiry_time_secs = expiry_time_secs

	# Keeps track of ports we've used, the creation key, and the last request
	# time for the port so they can be cached and cleaned up later.
	self._ports = {}

	# Locks port creation and cleanup. TODO(dalecurtis): If performance becomes
	# an issue a per-port based lock system can be used instead.
	self._port_lock = threading.Lock()

	def Get(self, key, new_port=False, **kwargs):
	"""Sets up a constrained port using the requested parameters.

	Requests for the same key and constraints will result in a cached port being
	returned if possible, subject to new_port.

	Args:
	key: Used to cache ports with the given constraints.
	new_port: Whether to create a new port or use an existing one if possible.
	**kwargs: Constraints to pass into traffic control.

	Returns:
	None if no port can be setup or the port number of the constrained port.
	"""
	with self._port_lock:
	# Check port key cache to see if this port is already setup. Update the
	# cache time and return the port if so. Performance isn't a concern here,
	# so just iterate over ports dict for simplicity.
	full_key = (key,) + tuple(kwargs.values())
	if not new_port:
	for port, status in self._ports.iteritems():
	if full_key == status['key']:
	self._ports[port]['last_update'] = time.time()
	return port

	# Cleanup ports on new port requests. Do it after the cache check though
	# so we don't erase and then setup the same port.
	if self._expiry_time_secs > 0:
	self._CleanupLocked(all_ports=False)

	# Performance isn't really an issue here, so just iterate over the port
	# range to find an unused port. If no port is found, None is returned.
	for port in xrange(self._port_range[0], self._port_range[1]):
	if port in self._ports:
	continue
	if self._SetupPort(port, **kwargs):
	kwargs['port'] = port
	self._ports[port] = {'last_update': time.time(), 'key': full_key,
	'config': kwargs}
	return port

	def _SetupPort(self, port, **kwargs):
	"""Setup network constraints on port using the requested parameters.

	Args:
	port: The port number to setup network constraints on.
	**kwargs: Network constraints to set up on the port.

	Returns:
	True if setting the network constraints on the port was successful, false
	otherwise.
	"""
	kwargs['port'] = port
	try:
	cherrypy.log('Setting up port %d' % port)
	traffic_control.CreateConstrainedPort(kwargs)
	return True
	except traffic_control.TrafficControlError as e:
	cherrypy.log('Error: %s\nOutput: %s' % (e.msg, e.error))
	return False

	def _CleanupLocked(self, all_ports):
	"""Internal cleanup method, expects lock to have already been acquired.

	See Cleanup() for more information.

	Args:
	all_ports: Should all ports be torn down regardless of expiration?
	"""
	now = time.time()
	# Use .items() instead of .iteritems() so we can delete keys w/o error.
	for port, status in self._ports.items():
	expired = now - status['last_update'] > self._expiry_time_secs
	if all_ports or expired:
	cherrypy.log('Cleaning up port %d' % port)
	self._DeletePort(port)
	del self._ports[port]

	def _DeletePort(self, port):
	"""Deletes network constraints on port.

	Args:
	port: The port number associated with the network constraints.
	"""
	try:
	traffic_control.DeleteConstrainedPort(self._ports[port]['config'])
	except traffic_control.TrafficControlError as e:
	cherrypy.log('Error: %s\nOutput: %s' % (e.msg, e.error))

	def Cleanup(self, interface, all_ports=False):
	"""Cleans up expired ports, or if all_ports=True, all allocated ports.

	By default, ports which haven't been used for self._expiry_time_secs are
	torn down. If all_ports=True then they are torn down regardless.

	Args:
	interface: Interface the constrained network is setup on.
	all_ports: Should all ports be torn down regardless of expiration?
	"""
	with self._port_lock:
	self._CleanupLocked(all_ports)


	class ConstrainedNetworkServer(object):
	"""A CherryPy-based HTTP server for serving files with network constraints."""

	def __init__(self, options, port_allocator):
	"""Sets up initial state for the CNS.

	Args:
	options: optparse based class returned by ParseArgs()
	port_allocator: A port allocator instance.
	"""
	self._options = options
	self._port_allocator = port_allocator

	@cherrypy.expose
	def ServeConstrained(self, f=None, bandwidth=None, latency=None, loss=None,
	new_port=False):
	"""Serves the requested file with the requested constraints.

	Subsequent requests for the same constraints from the same IP will share the
	previously created port unless new_port equals True. If no constraints
	are provided the file is served as is.

	Args:
	f: path relative to http root of file to serve.
	bandwidth: maximum allowed bandwidth for the provided port (integer
	in kbit/s).
	latency: time to add to each packet (integer in ms).
	loss: percentage of packets to drop (integer, 0-100).
	new_port: whether to use a new port for this request or not.
	"""
	cherrypy.log('Got request for %s, bandwidth=%s, latency=%s, loss=%s, '
	'new_port=%s' % (f, bandwidth, latency, loss, new_port))
	# CherryPy is a bit wonky at detecting parameters, so just make them all
	# optional and validate them ourselves.
	if not f:
	raise cherrypy.HTTPError(400, 'Invalid request. File must be specified.')

	# Sanitize and check the path to prevent www-root escapes.
	sanitized_path = os.path.abspath(os.path.join(self._options.www_root, f))
	if not sanitized_path.startswith(self._options.www_root):
	raise cherrypy.HTTPError(403, 'Invalid file requested.')

	# Check existence early to prevent wasted constraint setup.
	if not os.path.exists(sanitized_path):
	raise cherrypy.HTTPError(404, 'File not found.')

	# If there are no constraints, just serve the file.
	if bandwidth is None and latency is None and loss is None:
	return cherrypy.lib.static.serve_file(sanitized_path)

	# Validate inputs. isdigit() guarantees a natural number.
	bandwidth = self._ParseIntParameter(
	bandwidth, 'Invalid bandwidth constraint.', lambda x: x > 0)
	latency = self._ParseIntParameter(
	latency, 'Invalid latency constraint.', lambda x: x >= 0)
	loss = self._ParseIntParameter(
	loss, 'Invalid loss constraint.', lambda x: x <= 100 and x >= 0)

	# Allocate a port using the given constraints. If a port with the requested
	# key is already allocated, it will be reused.
	#
	# TODO(dalecurtis): The key cherrypy.request.remote.ip might not be unique
	# if build slaves are sharing the same VM.
	start_time = time.time()
	constrained_port = self._port_allocator.Get(
	cherrypy.request.remote.ip, server_port=self._options.port,
	interface=self._options.interface, bandwidth=bandwidth, latency=latency,
	loss=loss, new_port=new_port)
	end_time = time.time()

	if not constrained_port:
	raise cherrypy.HTTPError(503, 'Service unavailable. Out of ports.')

	cherrypy.log('Time to set up port %d = %ssec.' %
	(constrained_port, end_time - start_time))

	# Build constrained URL. Only pass on the file parameter.
	constrained_url = '%s?f=%s' % (
	cherrypy.url().replace(
	':%d' % self._options.port, ':%d' % constrained_port),
	f)

	# Redirect request to the constrained port.
	cherrypy.lib.cptools.redirect(constrained_url, internal=False)

	def _ParseIntParameter(self, param, msg, check):
	"""Returns integer value of param and verifies it satisfies the check.

	Args:
	param: Parameter name to check.
	msg: Message in error if raised.
	check: Check to verify the parameter value.

	Returns:
	None if param is None, integer value of param otherwise.

	Raises:
	cherrypy.HTTPError if param can not be converted to integer or if it does
	not satisfy the check.
	"""
	if param:
	try:
	int_value = int(param)
	if check(int_value):
	return int_value
	except:
	pass
	raise cherrypy.HTTPError(400, msg)


	def ParseArgs():
	"""Define and parse the command-line arguments."""
	parser = optparse.OptionParser()

	parser.add_option('--expiry-time', type='int',
	default=_DEFAULT_PORT_EXPIRY_TIME_SECS,
	help=('Number of seconds before constrained ports expire '
	'and are cleaned up. 0=Disabled. Default: %default'))
	parser.add_option('--port', type='int', default=_DEFAULT_SERVING_PORT,
	help='Port to serve the API on. Default: %default')
	parser.add_option('--port-range', default=_DEFAULT_CNS_PORT_RANGE,
	help=('Range of ports for constrained serving. Specify as '
	'a comma separated value pair. Default: %default'))
	parser.add_option('--interface', default='eth0',
	help=('Interface to setup constraints on. Use lo for a '
	'local client. Default: %default'))
	parser.add_option('--threads', type='int',
	default=cherrypy._cpserver.Server.thread_pool,
	help=('Number of threads in the thread pool. Default: '
	'%default'))
	parser.add_option('--www-root', default=os.getcwd(),
	help=('Directory root to serve files from. Defaults to the '
	'current directory: %default'))
	parser.add_option('-v', '--verbose', action='store_true', default=False,
	help='Turn on verbose output.')

	options = parser.parse_args()[0]

	# Convert port range into the desired tuple format.
	try:
	if isinstance(options.port_range, str):
	options.port_range = [int(port) for port in options.port_range.split(',')]
	except ValueError:
	parser.error('Invalid port range specified.')

	if options.expiry_time < 0:
	parser.error('Invalid expiry time specified.')

	# Convert the path to an absolute to remove any . or ..
	options.www_root = os.path.abspath(options.www_root)

	# Required so that cherrypy logs do not get propagated to root logger causing
	# the logs to be printed twice.
	cherrypy.log.error_log.propagate = False

	_SetLogger(options.verbose)

	return options


	def _SetLogger(verbose):
	# Logging is used for traffic_control debug statements.
	log_level = _DEFAULT_LOG_LEVEL
	if verbose:
	log_level = logging.DEBUG
	logging.basicConfig(level=log_level, format='[%(threadName)s] %(message)s')


	def Main():
	"""Configure and start the ConstrainedNetworkServer."""
	options = ParseArgs()

	try:
	traffic_control.CheckRequirements()
	except traffic_control.TrafficControlError as e:
	cherrypy.log(e.msg)
	return

	cherrypy.config.update(
	{'server.socket_host': '::', 'server.socket_port': options.port})

	if options.threads:
	cherrypy.config.update({'server.thread_pool': options.threads})

	# Setup port allocator here so we can call cleanup on failures/exit.
	pa = PortAllocator(options.port_range, expiry_time_secs=options.expiry_time)

	try:
	cherrypy.quickstart(ConstrainedNetworkServer(options, pa))
	finally:
	# Disable Ctrl-C handler to prevent interruption of cleanup.
	signal.signal(signal.SIGINT, lambda signal, frame: None)
	pa.Cleanup(options.interface, all_ports=True)


	if __name__ == '__main__':
	Main()