Added third_party python libraries that are needed for browser testing.
Review URL: http://codereview.chromium.org/6183003
git-svn-id: svn://svn.chromium.org/native_client/trunk/src/third_party/pylib@4105 fcba33aa-ac0c-11dd-b9e7-8d5594d729c2
diff --git a/mozrunner/LICENSE b/mozrunner/LICENSE
new file mode 100644
index 0000000..a21b684
--- /dev/null
+++ b/mozrunner/LICENSE
@@ -0,0 +1,35 @@
+Version: MPL 1.1/GPL 2.0/LGPL 2.1
+
+The contents of this file are subject to the Mozilla Public License Version
+1.1 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+http://www.mozilla.org/MPL/
+
+Software distributed under the License is distributed on an "AS IS" basis,
+WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+for the specific language governing rights and limitations under the
+License.
+
+The Original Code is Mozilla Corporation Code.
+
+The Initial Developer of the Original Code is
+Mikeal Rogers.
+Portions created by the Initial Developer are Copyright (C) 2008-2009
+the Initial Developer. All Rights Reserved.
+
+Contributor(s):
+ Mikeal Rogers <mikeal.rogers@gmail.com>
+ Clint Talbert <ctalbert@mozilla.com>
+ Henrik Skupin <hskupin@mozilla.com>
+
+Alternatively, the contents of this file may be used under the terms of
+either the GNU General Public License Version 2 or later (the "GPL"), or
+the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+in which case the provisions of the GPL or the LGPL are applicable instead
+of those above. If you wish to allow use of your version of this file only
+under the terms of either the GPL or the LGPL, and not to allow others to
+use your version of this file under the terms of the MPL, indicate your
+decision by deleting the provisions above and replace them with the notice
+and other provisions required by the GPL or the LGPL. If you do not delete
+the provisions above, a recipient may use your version of this file under
+the terms of any one of the MPL, the GPL or the LGPL.
diff --git a/mozrunner/README b/mozrunner/README
new file mode 100644
index 0000000..e4e8298
--- /dev/null
+++ b/mozrunner/README
@@ -0,0 +1,12 @@
+URL: http://pypi.python.org/packages/source/m/mozrunner/mozrunner-2.5.2.tar.gz#md5=c8c21ecfc0dcc78c06865e611cf1d153
+Version: 2.5.2
+License: MPL 1.1/GPL 2.0/LGPL 2.1
+License File: LICENSE
+
+Description:
+mozrunner is a Python module for running Mozilla applications such as Firefox.
+
+
+
+Local Modifications:
+No Modifications
diff --git a/mozrunner/__init__.py b/mozrunner/__init__.py
new file mode 100644
index 0000000..2f19a2c
--- /dev/null
+++ b/mozrunner/__init__.py
@@ -0,0 +1,639 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla Corporation Code.
+#
+# The Initial Developer of the Original Code is
+# Mikeal Rogers.
+# Portions created by the Initial Developer are Copyright (C) 2008-2009
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+# Mikeal Rogers <mikeal.rogers@gmail.com>
+# Clint Talbert <ctalbert@mozilla.com>
+# Henrik Skupin <hskupin@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os
+import sys
+import copy
+import tempfile
+import signal
+import commands
+import zipfile
+import optparse
+import killableprocess
+import subprocess
+import platform
+
+from distutils import dir_util
+from time import sleep
+from xml.dom import minidom
+
+# conditional (version-dependent) imports
+try:
+ import simplejson
+except ImportError:
+ import json as simplejson
+
+import logging
+logger = logging.getLogger(__name__)
+
+# Use dir_util for copy/rm operations because shutil is all kinds of broken
+copytree = dir_util.copy_tree
+rmtree = dir_util.remove_tree
+
+def findInPath(fileName, path=os.environ['PATH']):
+ dirs = path.split(os.pathsep)
+ for dir in dirs:
+ if os.path.isfile(os.path.join(dir, fileName)):
+ return os.path.join(dir, fileName)
+ if os.name == 'nt' or sys.platform == 'cygwin':
+ if os.path.isfile(os.path.join(dir, fileName + ".exe")):
+ return os.path.join(dir, fileName + ".exe")
+ return None
+
+stdout = sys.stdout
+stderr = sys.stderr
+stdin = sys.stdin
+
+def run_command(cmd, env=None, **kwargs):
+ """Run the given command in killable process."""
+ killable_kwargs = {'stdout':stdout ,'stderr':stderr, 'stdin':stdin}
+ killable_kwargs.update(kwargs)
+
+ if sys.platform != "win32":
+ return killableprocess.Popen(cmd, preexec_fn=lambda : os.setpgid(0, 0),
+ env=env, **killable_kwargs)
+ else:
+ return killableprocess.Popen(cmd, env=env, **killable_kwargs)
+
+def getoutput(l):
+ tmp = tempfile.mktemp()
+ x = open(tmp, 'w')
+ subprocess.call(l, stdout=x, stderr=x)
+ x.close(); x = open(tmp, 'r')
+ r = x.read() ; x.close()
+ os.remove(tmp)
+ return r
+
+def get_pids(name, minimun_pid=0):
+ """Get all the pids matching name, exclude any pids below minimum_pid."""
+ if os.name == 'nt' or sys.platform == 'cygwin':
+ import wpk
+
+ pids = wpk.get_pids(name)
+
+ else:
+ # get_pids_cmd = ['ps', 'ax']
+ # h = killableprocess.runCommand(get_pids_cmd, stdout=subprocess.PIPE, universal_newlines=True)
+ # h.wait(group=False)
+ # data = h.stdout.readlines()
+ data = getoutput(['ps', 'ax']).splitlines()
+ pids = [int(line.split()[0]) for line in data if line.find(name) is not -1]
+
+ matching_pids = [m for m in pids if m > minimun_pid]
+ return matching_pids
+
+def kill_process_by_name(name):
+ """Find and kill all processes containing a certain name"""
+
+ pids = get_pids(name)
+
+ if os.name == 'nt' or sys.platform == 'cygwin':
+ for p in pids:
+ import wpk
+
+ wpk.kill_pid(p)
+
+ else:
+ for pid in pids:
+ try:
+ os.kill(pid, signal.SIGTERM)
+ except OSError: pass
+ sleep(.5)
+ if len(get_pids(name)) is not 0:
+ try:
+ os.kill(pid, signal.SIGKILL)
+ except OSError: pass
+ sleep(.5)
+ if len(get_pids(name)) is not 0:
+ logger.error('Could not kill process')
+
+def makedirs(name):
+
+ head, tail = os.path.split(name)
+ if not tail:
+ head, tail = os.path.split(head)
+ if head and tail and not os.path.exists(head):
+ try:
+ makedirs(head)
+ except OSError, e:
+ pass
+ if tail == os.curdir: # xxx/newdir/. exists if xxx/newdir exists
+ return
+ try:
+ os.mkdir(name)
+ except:
+ pass
+
+class Profile(object):
+ """Handles all operations regarding profile. Created new profiles, installs extensions,
+ sets preferences and handles cleanup."""
+
+ def __init__(self, binary=None, profile=None, addons=None,
+ preferences=None):
+
+ self.binary = binary
+
+ self.create_new = not(bool(profile))
+ if profile:
+ self.profile = profile
+ else:
+ self.profile = self.create_new_profile(self.binary)
+
+ self.addons_installed = []
+ self.addons = addons or []
+
+ ### set preferences from class preferences
+ preferences = preferences or {}
+ if hasattr(self.__class__, 'preferences'):
+ self.preferences = self.__class__.preferences.copy()
+ else:
+ self.preferences = {}
+ self.preferences.update(preferences)
+
+ for addon in self.addons:
+ self.install_addon(addon)
+
+ self.set_preferences(self.preferences)
+
+ def create_new_profile(self, binary):
+ """Create a new clean profile in tmp which is a simple empty folder"""
+ profile = tempfile.mkdtemp(suffix='.mozrunner')
+ return profile
+
+ ### methods related to addons
+
+ @classmethod
+ def addon_id(self, addon_path):
+ """
+ return the id for a given addon, or None if not found
+ - addon_path : path to the addon directory
+ """
+
+ def find_id(desc):
+ """finds the addon id give its description"""
+
+ addon_id = None
+ for elem in desc:
+ apps = elem.getElementsByTagName('em:targetApplication')
+ if apps:
+ for app in apps:
+ # remove targetApplication nodes, they contain id's we aren't interested in
+ elem.removeChild(app)
+ if elem.getElementsByTagName('em:id'):
+ addon_id = str(elem.getElementsByTagName('em:id')[0].firstChild.data)
+ elif elem.hasAttribute('em:id'):
+ addon_id = str(elem.getAttribute('em:id'))
+ return addon_id
+
+ doc = minidom.parse(os.path.join(addon_path, 'install.rdf'))
+
+ for tag in 'Description', 'RDF:Description':
+ desc = doc.getElementsByTagName(tag)
+ addon_id = find_id(desc)
+ if addon_id:
+ return addon_id
+
+
+ def install_addon(self, path):
+ """Installs the given addon or directory of addons in the profile."""
+
+ # if the addon is a directory, install all addons in it
+ addons = [path]
+ if not path.endswith('.xpi') and not os.path.exists(os.path.join(path, 'install.rdf')):
+ addons = [os.path.join(path, x) for x in os.listdir(path)]
+
+ # install each addon
+ for addon in addons:
+
+ # if the addon is an .xpi, uncompress it to a temporary directory
+ if addon.endswith('.xpi'):
+ tmpdir = tempfile.mkdtemp(suffix = "." + os.path.split(addon)[-1])
+ compressed_file = zipfile.ZipFile(addon, "r")
+ for name in compressed_file.namelist():
+ if name.endswith('/'):
+ makedirs(os.path.join(tmpdir, name))
+ else:
+ if not os.path.isdir(os.path.dirname(os.path.join(tmpdir, name))):
+ makedirs(os.path.dirname(os.path.join(tmpdir, name)))
+ data = compressed_file.read(name)
+ f = open(os.path.join(tmpdir, name), 'wb')
+ f.write(data) ; f.close()
+ addon = tmpdir
+
+ # determine the addon id
+ addon_id = Profile.addon_id(addon)
+ assert addon_id is not None, "The addon id could not be found: %s" % addon
+
+ # copy the addon to the profile
+ addon_path = os.path.join(self.profile, 'extensions', addon_id)
+ copytree(addon, addon_path, preserve_symlinks=1)
+ self.addons_installed.append(addon_path)
+
+ def clean_addons(self):
+ """Cleans up addons in the profile."""
+ for addon in self.addons_installed:
+ if os.path.isdir(addon):
+ rmtree(addon)
+
+ ### methods related to preferences
+
+ def set_preferences(self, preferences):
+ """Adds preferences dict to profile preferences"""
+
+ prefs_file = os.path.join(self.profile, 'user.js')
+
+ # Ensure that the file exists first otherwise create an empty file
+ if os.path.isfile(prefs_file):
+ f = open(prefs_file, 'a+')
+ else:
+ f = open(prefs_file, 'w')
+
+ f.write('\n#MozRunner Prefs Start\n')
+
+ pref_lines = ['user_pref(%s, %s);' %
+ (simplejson.dumps(k), simplejson.dumps(v) ) for k, v in
+ preferences.items()]
+ for line in pref_lines:
+ f.write(line+'\n')
+ f.write('#MozRunner Prefs End\n')
+ f.flush() ; f.close()
+
+ def clean_preferences(self):
+ """Removed preferences added by mozrunner."""
+ lines = open(os.path.join(self.profile, 'user.js'), 'r').read().splitlines()
+ s = lines.index('#MozRunner Prefs Start') ; e = lines.index('#MozRunner Prefs End')
+ cleaned_prefs = '\n'.join(lines[:s] + lines[e+1:])
+ f = open(os.path.join(self.profile, 'user.js'), 'w')
+ f.write(cleaned_prefs) ; f.flush() ; f.close()
+
+ ### cleanup
+
+ def cleanup(self):
+ """Cleanup operations on the profile."""
+ if self.create_new:
+ rmtree(self.profile)
+ else:
+ self.clean_preferences()
+ self.clean_addons()
+
+class FirefoxProfile(Profile):
+ """Specialized Profile subclass for Firefox"""
+ preferences = {# Don't automatically update the application
+ 'app.update.enabled' : False,
+ # Don't restore the last open set of tabs if the browser has crashed
+ 'browser.sessionstore.resume_from_crash': False,
+ # Don't check for the default web browser
+ 'browser.shell.checkDefaultBrowser' : False,
+ # Don't warn on exit when multiple tabs are open
+ 'browser.tabs.warnOnClose' : False,
+ # Don't warn when exiting the browser
+ 'browser.warnOnQuit': False,
+ # Only install add-ons from the profile and the app folder
+ 'extensions.enabledScopes' : 5,
+ # Dont' run the add-on compatibility check during start-up
+ 'extensions.showMismatchUI' : False,
+ # Don't automatically update add-ons
+ 'extensions.update.enabled' : False,
+ # Don't open a dialog to show available add-on updates
+ 'extensions.update.notifyUser' : False,
+ }
+
+ @property
+ def names(self):
+ if sys.platform == 'darwin':
+ return ['firefox', 'minefield', 'shiretoko']
+ if (sys.platform == 'linux2') or (sys.platform in ('sunos5', 'solaris')):
+ return ['firefox', 'mozilla-firefox', 'iceweasel']
+ if os.name == 'nt' or sys.platform == 'cygwin':
+ return ['firefox']
+
+class ThunderbirdProfile(Profile):
+ preferences = {'extensions.update.enabled' : False,
+ 'extensions.update.notifyUser' : False,
+ 'browser.shell.checkDefaultBrowser' : False,
+ 'browser.tabs.warnOnClose' : False,
+ 'browser.warnOnQuit': False,
+ 'browser.sessionstore.resume_from_crash': False,
+ }
+ names = ["thunderbird", "shredder"]
+
+
+class Runner(object):
+ """Handles all running operations. Finds bins, runs and kills the process."""
+
+ def __init__(self, binary=None, profile=None, cmdargs=[], env=None,
+ aggressively_kill=['crashreporter'], kp_kwargs={}):
+ if binary is None:
+ self.binary = self.find_binary()
+ elif sys.platform == 'darwin' and binary.find('Contents/MacOS/') == -1:
+ self.binary = os.path.join(binary, 'Contents/MacOS/%s-bin' % self.names[0])
+ else:
+ self.binary = binary
+
+ if not os.path.exists(self.binary):
+ raise Exception("Binary path does not exist "+self.binary)
+
+ if sys.platform == 'linux2' and self.binary.endswith('-bin'):
+ dirname = os.path.dirname(self.binary)
+ if os.environ.get('LD_LIBRARY_PATH', None):
+ os.environ['LD_LIBRARY_PATH'] = '%s:%s' % (os.environ['LD_LIBRARY_PATH'], dirname)
+ else:
+ os.environ['LD_LIBRARY_PATH'] = dirname
+
+ self.profile = profile
+
+ self.cmdargs = cmdargs
+ if env is None:
+ self.env = copy.copy(os.environ)
+ self.env.update({'MOZ_NO_REMOTE':"1",})
+ else:
+ self.env = env
+ self.aggressively_kill = aggressively_kill
+ self.kp_kwargs = kp_kwargs
+
+ def find_binary(self):
+ """Finds the binary for self.names if one was not provided."""
+ binary = None
+ if sys.platform in ('linux2', 'sunos5', 'solaris'):
+ for name in reversed(self.names):
+ binary = findInPath(name)
+ elif os.name == 'nt' or sys.platform == 'cygwin':
+
+ # find the default executable from the windows registry
+ try:
+ # assumes self.app_name is defined, as it should be for
+ # implementors
+ import _winreg
+ app_key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r"Software\Mozilla\Mozilla %s" % self.app_name)
+ version, _type = _winreg.QueryValueEx(app_key, "CurrentVersion")
+ version_key = _winreg.OpenKey(app_key, version + r"\Main")
+ path, _ = _winreg.QueryValueEx(version_key, "PathToExe")
+ return path
+ except: # XXX not sure what type of exception this should be
+ pass
+
+ # search for the binary in the path
+ for name in reversed(self.names):
+ binary = findInPath(name)
+ if sys.platform == 'cygwin':
+ program_files = os.environ['PROGRAMFILES']
+ else:
+ program_files = os.environ['ProgramFiles']
+
+ if binary is None:
+ for bin in [(program_files, 'Mozilla Firefox', 'firefox.exe'),
+ (os.environ.get("ProgramFiles(x86)"),'Mozilla Firefox', 'firefox.exe'),
+ (program_files,'Minefield', 'firefox.exe'),
+ (os.environ.get("ProgramFiles(x86)"),'Minefield', 'firefox.exe')
+ ]:
+ path = os.path.join(*bin)
+ if os.path.isfile(path):
+ binary = path
+ break
+ elif sys.platform == 'darwin':
+ for name in reversed(self.names):
+ appdir = os.path.join('Applications', name.capitalize()+'.app')
+ if os.path.isdir(os.path.join(os.path.expanduser('~/'), appdir)):
+ binary = os.path.join(os.path.expanduser('~/'), appdir,
+ 'Contents/MacOS/'+name+'-bin')
+ elif os.path.isdir('/'+appdir):
+ binary = os.path.join("/"+appdir, 'Contents/MacOS/'+name+'-bin')
+
+ if binary is not None:
+ if not os.path.isfile(binary):
+ binary = binary.replace(name+'-bin', 'firefox-bin')
+ if not os.path.isfile(binary):
+ binary = None
+ if binary is None:
+ raise Exception('Mozrunner could not locate your binary, you will need to set it.')
+ return binary
+
+ @property
+ def command(self):
+ """Returns the command list to run."""
+ cmd = [self.binary, '-profile', self.profile.profile]
+ # On i386 OS X machines, i386+x86_64 universal binaries need to be told
+ # to run as i386 binaries. If we're not running a i386+x86_64 universal
+ # binary, then this command modification is harmless.
+ if sys.platform == 'darwin':
+ if hasattr(platform, 'architecture') and platform.architecture()[0] == '32bit':
+ cmd = ['arch', '-i386'] + cmd
+ return cmd
+
+ def get_repositoryInfo(self):
+ """Read repository information from application.ini and platform.ini."""
+ import ConfigParser
+
+ config = ConfigParser.RawConfigParser()
+ dirname = os.path.dirname(self.binary)
+ repository = { }
+
+ for entry in [['application', 'App'], ['platform', 'Build']]:
+ (file, section) = entry
+ config.read(os.path.join(dirname, '%s.ini' % file))
+
+ for entry in [['SourceRepository', 'repository'], ['SourceStamp', 'changeset']]:
+ (key, id) = entry
+
+ try:
+ repository['%s_%s' % (file, id)] = config.get(section, key);
+ except:
+ repository['%s_%s' % (file, id)] = None
+
+ return repository
+
+ def start(self):
+ """Run self.command in the proper environment."""
+ if self.profile is None:
+ self.profile = self.profile_class()
+ self.process_handler = run_command(self.command+self.cmdargs, self.env, **self.kp_kwargs)
+
+ def wait(self, timeout=None):
+ """Wait for the browser to exit."""
+ self.process_handler.wait(timeout=timeout)
+
+ if sys.platform != 'win32':
+ for name in self.names:
+ for pid in get_pids(name, self.process_handler.pid):
+ self.process_handler.pid = pid
+ self.process_handler.wait(timeout=timeout)
+
+ def kill(self, kill_signal=signal.SIGTERM):
+ """Kill the browser"""
+ if sys.platform != 'win32':
+ self.process_handler.kill()
+ for name in self.names:
+ for pid in get_pids(name, self.process_handler.pid):
+ self.process_handler.pid = pid
+ self.process_handler.kill()
+ else:
+ try:
+ self.process_handler.kill(group=True)
+ except Exception, e:
+ logger.error('Cannot kill process, '+type(e).__name__+' '+e.message)
+
+ for name in self.aggressively_kill:
+ kill_process_by_name(name)
+
+ def stop(self):
+ self.kill()
+
+class FirefoxRunner(Runner):
+ """Specialized Runner subclass for running Firefox."""
+
+ app_name = 'Firefox'
+ profile_class = FirefoxProfile
+
+ @property
+ def names(self):
+ if sys.platform == 'darwin':
+ return ['firefox', 'minefield', 'shiretoko']
+ if (sys.platform == 'linux2') or (sys.platform in ('sunos5', 'solaris')):
+ return ['firefox', 'mozilla-firefox', 'iceweasel']
+ if os.name == 'nt' or sys.platform == 'cygwin':
+ return ['firefox']
+
+class ThunderbirdRunner(Runner):
+ """Specialized Runner subclass for running Thunderbird"""
+
+ app_name = 'Thunderbird'
+ profile_class = ThunderbirdProfile
+
+ names = ["thunderbird", "shredder"]
+
+class CLI(object):
+ """Command line interface."""
+
+ runner_class = FirefoxRunner
+ profile_class = FirefoxProfile
+ module = "mozrunner"
+
+ parser_options = {("-b", "--binary",): dict(dest="binary", help="Binary path.",
+ metavar=None, default=None),
+ ('-p', "--profile",): dict(dest="profile", help="Profile path.",
+ metavar=None, default=None),
+ ('-a', "--addons",): dict(dest="addons",
+ help="Addons paths to install.",
+ metavar=None, default=None),
+ ("--info",): dict(dest="info", default=False,
+ action="store_true",
+ help="Print module information")
+ }
+
+ def __init__(self):
+ """ Setup command line parser and parse arguments """
+ self.metadata = self.get_metadata_from_egg()
+ self.parser = optparse.OptionParser(version="%prog " + self.metadata["Version"])
+ for names, opts in self.parser_options.items():
+ self.parser.add_option(*names, **opts)
+ (self.options, self.args) = self.parser.parse_args()
+
+ if self.options.info:
+ self.print_metadata()
+ sys.exit(0)
+
+ # XXX should use action='append' instead of rolling our own
+ try:
+ self.addons = self.options.addons.split(',')
+ except:
+ self.addons = []
+
+ def get_metadata_from_egg(self):
+ import pkg_resources
+ ret = {}
+ dist = pkg_resources.get_distribution(self.module)
+ if dist.has_metadata("PKG-INFO"):
+ for line in dist.get_metadata_lines("PKG-INFO"):
+ key, value = line.split(':', 1)
+ ret[key] = value
+ if dist.has_metadata("requires.txt"):
+ ret["Dependencies"] = "\n" + dist.get_metadata("requires.txt")
+ return ret
+
+ def print_metadata(self, data=("Name", "Version", "Summary", "Home-page",
+ "Author", "Author-email", "License", "Platform", "Dependencies")):
+ for key in data:
+ if key in self.metadata:
+ print key + ": " + self.metadata[key]
+
+ def create_runner(self):
+ """ Get the runner object """
+ runner = self.get_runner(binary=self.options.binary)
+ profile = self.get_profile(binary=runner.binary,
+ profile=self.options.profile,
+ addons=self.addons)
+ runner.profile = profile
+ return runner
+
+ def get_runner(self, binary=None, profile=None):
+ """Returns the runner instance for the given command line binary argument
+ the profile instance returned from self.get_profile()."""
+ return self.runner_class(binary, profile)
+
+ def get_profile(self, binary=None, profile=None, addons=None, preferences=None):
+ """Returns the profile instance for the given command line arguments."""
+ addons = addons or []
+ preferences = preferences or {}
+ return self.profile_class(binary, profile, addons, preferences)
+
+ def run(self):
+ runner = self.create_runner()
+ self.start(runner)
+ runner.profile.cleanup()
+
+ def start(self, runner):
+ """Starts the runner and waits for Firefox to exitor Keyboard Interrupt.
+ Shoule be overwritten to provide custom running of the runner instance."""
+ runner.start()
+ print 'Started:', ' '.join(runner.command)
+ try:
+ runner.wait()
+ except KeyboardInterrupt:
+ runner.stop()
+
+
+def cli():
+ CLI().run()
+
+def print_addon_ids(args=sys.argv[1:]):
+ """print addon ids for testing"""
+ for arg in args:
+ print Profile.addon_id(arg)
+
+
diff --git a/mozrunner/killableprocess.py b/mozrunner/killableprocess.py
new file mode 100644
index 0000000..dcc91ff
--- /dev/null
+++ b/mozrunner/killableprocess.py
@@ -0,0 +1,292 @@
+# killableprocess - subprocesses which can be reliably killed
+#
+# Parts of this module are copied from the subprocess.py file contained
+# in the Python distribution.
+#
+# Copyright (c) 2003-2004 by Peter Astrand <astrand@lysator.liu.se>
+#
+# Additions and modifications written by Benjamin Smedberg
+# <benjamin@smedbergs.us> are Copyright (c) 2006 by the Mozilla Foundation
+# <http://www.mozilla.org/>
+#
+# More Modifications
+# Copyright (c) 2006-2007 by Mike Taylor <bear@code-bear.com>
+# Copyright (c) 2007-2008 by Mikeal Rogers <mikeal@mozilla.com>
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of the
+# author not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
+# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""killableprocess - Subprocesses which can be reliably killed
+
+This module is a subclass of the builtin "subprocess" module. It allows
+processes that launch subprocesses to be reliably killed on Windows (via the Popen.kill() method.
+
+It also adds a timeout argument to Wait() for a limited period of time before
+forcefully killing the process.
+
+Note: On Windows, this module requires Windows 2000 or higher (no support for
+Windows 95, 98, or NT 4.0). It also requires ctypes, which is bundled with
+Python 2.5+ or available from http://python.net/crew/theller/ctypes/
+"""
+
+import subprocess
+import sys
+import os
+import time
+import datetime
+import types
+import exceptions
+
+try:
+ from subprocess import CalledProcessError
+except ImportError:
+ # Python 2.4 doesn't implement CalledProcessError
+ class CalledProcessError(Exception):
+ """This exception is raised when a process run by check_call() returns
+ a non-zero exit status. The exit status will be stored in the
+ returncode attribute."""
+ def __init__(self, returncode, cmd):
+ self.returncode = returncode
+ self.cmd = cmd
+ def __str__(self):
+ return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
+
+mswindows = (sys.platform == "win32")
+
+if mswindows:
+ import winprocess
+else:
+ import signal
+
+def call(*args, **kwargs):
+ waitargs = {}
+ if "timeout" in kwargs:
+ waitargs["timeout"] = kwargs.pop("timeout")
+
+ return Popen(*args, **kwargs).wait(**waitargs)
+
+def check_call(*args, **kwargs):
+ """Call a program with an optional timeout. If the program has a non-zero
+ exit status, raises a CalledProcessError."""
+
+ retcode = call(*args, **kwargs)
+ if retcode:
+ cmd = kwargs.get("args")
+ if cmd is None:
+ cmd = args[0]
+ raise CalledProcessError(retcode, cmd)
+
+if not mswindows:
+ def DoNothing(*args):
+ pass
+
+class Popen(subprocess.Popen):
+ kill_called = False
+ if mswindows:
+ def _execute_child(self, args, executable, preexec_fn, close_fds,
+ cwd, env, universal_newlines, startupinfo,
+ creationflags, shell,
+ p2cread, p2cwrite,
+ c2pread, c2pwrite,
+ errread, errwrite):
+ if not isinstance(args, types.StringTypes):
+ args = subprocess.list2cmdline(args)
+
+ # Always or in the create new process group
+ creationflags |= winprocess.CREATE_NEW_PROCESS_GROUP
+
+ if startupinfo is None:
+ startupinfo = winprocess.STARTUPINFO()
+
+ if None not in (p2cread, c2pwrite, errwrite):
+ startupinfo.dwFlags |= winprocess.STARTF_USESTDHANDLES
+
+ startupinfo.hStdInput = int(p2cread)
+ startupinfo.hStdOutput = int(c2pwrite)
+ startupinfo.hStdError = int(errwrite)
+ if shell:
+ startupinfo.dwFlags |= winprocess.STARTF_USESHOWWINDOW
+ startupinfo.wShowWindow = winprocess.SW_HIDE
+ comspec = os.environ.get("COMSPEC", "cmd.exe")
+ args = comspec + " /c " + args
+
+ # determine if we can create create a job
+ canCreateJob = winprocess.CanCreateJobObject()
+
+ # set process creation flags
+ creationflags |= winprocess.CREATE_SUSPENDED
+ creationflags |= winprocess.CREATE_UNICODE_ENVIRONMENT
+ if canCreateJob:
+ creationflags |= winprocess.CREATE_BREAKAWAY_FROM_JOB
+
+ # create the process
+ hp, ht, pid, tid = winprocess.CreateProcess(
+ executable, args,
+ None, None, # No special security
+ 1, # Must inherit handles!
+ creationflags,
+ winprocess.EnvironmentBlock(env),
+ cwd, startupinfo)
+ self._child_created = True
+ self._handle = hp
+ self._thread = ht
+ self.pid = pid
+ self.tid = tid
+
+ if canCreateJob:
+ # We create a new job for this process, so that we can kill
+ # the process and any sub-processes
+ self._job = winprocess.CreateJobObject()
+ winprocess.AssignProcessToJobObject(self._job, int(hp))
+ else:
+ self._job = None
+
+ winprocess.ResumeThread(int(ht))
+ ht.Close()
+
+ if p2cread is not None:
+ p2cread.Close()
+ if c2pwrite is not None:
+ c2pwrite.Close()
+ if errwrite is not None:
+ errwrite.Close()
+ time.sleep(.1)
+
+ def kill(self, group=True):
+ """Kill the process. If group=True, all sub-processes will also be killed."""
+ self.kill_called = True
+ if mswindows:
+ if group and self._job:
+ winprocess.TerminateJobObject(self._job, 127)
+ else:
+ try:
+ winprocess.TerminateProcess(self._handle, 127)
+ except:
+ # TODO: better error handling here
+ pass
+ self.returncode = 127
+ else:
+ if group:
+ try:
+ os.killpg(self.pid, signal.SIGKILL)
+ except: pass
+ else:
+ os.kill(self.pid, signal.SIGKILL)
+ self.returncode = -9
+
+ def wait(self, timeout=None, group=True):
+ """Wait for the process to terminate. Returns returncode attribute.
+ If timeout seconds are reached and the process has not terminated,
+ it will be forcefully killed. If timeout is -1, wait will not
+ time out."""
+
+ if timeout is not None:
+ # timeout is now in milliseconds
+ timeout = timeout * 1000
+
+ if self.returncode is not None:
+ return self.returncode
+
+ starttime = datetime.datetime.now()
+
+ if mswindows:
+ if timeout is None:
+ timeout = -1
+ rc = winprocess.WaitForSingleObject(self._handle, timeout)
+
+ if rc != winprocess.WAIT_TIMEOUT:
+ def check():
+ now = datetime.datetime.now()
+ diff = now - starttime
+ if (diff.seconds * 1000 * 1000 + diff.microseconds) < (timeout * 1000):
+ if self._job:
+ if (winprocess.QueryInformationJobObject(self._job, 8)['BasicInfo']['ActiveProcesses'] > 0):
+ return True
+ else:
+ return True
+ return False
+ while check():
+ time.sleep(.5)
+
+ now = datetime.datetime.now()
+ diff = now - starttime
+ if (diff.seconds * 1000 * 1000 + diff.microseconds) > (timeout * 1000):
+ self.kill(group)
+ else:
+ self.returncode = winprocess.GetExitCodeProcess(self._handle)
+ else:
+ if (sys.platform == 'linux2') or (sys.platform in ('sunos5', 'solaris')):
+ def group_wait(timeout):
+ try:
+ os.waitpid(self.pid, 0)
+ except OSError, e:
+ pass # If wait has already been called on this pid, bad things happen
+ return self.returncode
+ elif sys.platform == 'darwin':
+ def group_wait(timeout):
+ try:
+ count = 0
+ if timeout is None and self.kill_called:
+ timeout = 10 # Have to set some kind of timeout or else this could go on forever
+ if timeout is None:
+ while 1:
+ os.killpg(self.pid, signal.SIG_DFL)
+ while ((count * 2) <= timeout):
+ os.killpg(self.pid, signal.SIG_DFL)
+ # count is increased by 500ms for every 0.5s of sleep
+ time.sleep(.5); count += 500
+ except exceptions.OSError:
+ return self.returncode
+
+ if timeout is None:
+ if group is True:
+ return group_wait(timeout)
+ else:
+ subprocess.Popen.wait(self)
+ return self.returncode
+
+ returncode = False
+
+ now = datetime.datetime.now()
+ diff = now - starttime
+ while (diff.seconds * 1000 * 1000 + diff.microseconds) < (timeout * 1000) and ( returncode is False ):
+ if group is True:
+ return group_wait(timeout)
+ else:
+ if subprocess.poll() is not None:
+ returncode = self.returncode
+ time.sleep(.5)
+ now = datetime.datetime.now()
+ diff = now - starttime
+ return self.returncode
+
+ return self.returncode
+ # We get random maxint errors from subprocesses __del__
+ __del__ = lambda self: None
+
+def setpgid_preexec_fn():
+ os.setpgid(0, 0)
+
+def runCommand(cmd, **kwargs):
+ if sys.platform != "win32":
+ return Popen(cmd, preexec_fn=setpgid_preexec_fn, **kwargs)
+ else:
+ return Popen(cmd, **kwargs)
diff --git a/mozrunner/qijo.py b/mozrunner/qijo.py
new file mode 100644
index 0000000..d4da88c
--- /dev/null
+++ b/mozrunner/qijo.py
@@ -0,0 +1,162 @@
+from ctypes import c_void_p, POINTER, sizeof, Structure, windll, WinError, WINFUNCTYPE, addressof, c_size_t, c_ulong
+from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LARGE_INTEGER
+
+LPVOID = c_void_p
+LPDWORD = POINTER(DWORD)
+SIZE_T = c_size_t
+ULONG_PTR = POINTER(c_ulong)
+
+# A ULONGLONG is a 64-bit unsigned integer.
+# Thus there are 8 bytes in a ULONGLONG.
+# XXX why not import c_ulonglong ?
+ULONGLONG = BYTE * 8
+
+class IO_COUNTERS(Structure):
+ # The IO_COUNTERS struct is 6 ULONGLONGs.
+ # TODO: Replace with non-dummy fields.
+ _fields_ = [('dummy', ULONGLONG * 6)]
+
+class JOBOBJECT_BASIC_ACCOUNTING_INFORMATION(Structure):
+ _fields_ = [('TotalUserTime', LARGE_INTEGER),
+ ('TotalKernelTime', LARGE_INTEGER),
+ ('ThisPeriodTotalUserTime', LARGE_INTEGER),
+ ('ThisPeriodTotalKernelTime', LARGE_INTEGER),
+ ('TotalPageFaultCount', DWORD),
+ ('TotalProcesses', DWORD),
+ ('ActiveProcesses', DWORD),
+ ('TotalTerminatedProcesses', DWORD)]
+
+class JOBOBJECT_BASIC_AND_IO_ACCOUNTING_INFORMATION(Structure):
+ _fields_ = [('BasicInfo', JOBOBJECT_BASIC_ACCOUNTING_INFORMATION),
+ ('IoInfo', IO_COUNTERS)]
+
+# see http://msdn.microsoft.com/en-us/library/ms684147%28VS.85%29.aspx
+class JOBOBJECT_BASIC_LIMIT_INFORMATION(Structure):
+ _fields_ = [('PerProcessUserTimeLimit', LARGE_INTEGER),
+ ('PerJobUserTimeLimit', LARGE_INTEGER),
+ ('LimitFlags', DWORD),
+ ('MinimumWorkingSetSize', SIZE_T),
+ ('MaximumWorkingSetSize', SIZE_T),
+ ('ActiveProcessLimit', DWORD),
+ ('Affinity', ULONG_PTR),
+ ('PriorityClass', DWORD),
+ ('SchedulingClass', DWORD)
+ ]
+
+# see http://msdn.microsoft.com/en-us/library/ms684156%28VS.85%29.aspx
+class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(Structure):
+ _fields_ = [('BasicLimitInformation', JOBOBJECT_BASIC_LIMIT_INFORMATION),
+ ('IoInfo', IO_COUNTERS),
+ ('ProcessMemoryLimit', SIZE_T),
+ ('JobMemoryLimit', SIZE_T),
+ ('PeakProcessMemoryUsed', SIZE_T),
+ ('PeakJobMemoryUsed', SIZE_T)]
+
+# XXX Magical numbers like 8 should be documented
+JobObjectBasicAndIoAccountingInformation = 8
+
+# ...like magical number 9 comes from
+# http://community.flexerasoftware.com/archive/index.php?t-181670.html
+# I wish I had a more canonical source
+JobObjectExtendedLimitInformation = 9
+
+class JobObjectInfo(object):
+ mapping = { 'JobObjectBasicAndIoAccountingInformation': 8,
+ 'JobObjectExtendedLimitInformation': 9
+ }
+ structures = { 8: JOBOBJECT_BASIC_AND_IO_ACCOUNTING_INFORMATION,
+ 9: JOBOBJECT_EXTENDED_LIMIT_INFORMATION
+ }
+ def __init__(self, _class):
+ if isinstance(_class, basestring):
+ assert _class in self.mapping, 'Class should be one of %s; you gave %s' % (self.mapping, _class)
+ _class = self.mapping[_class]
+ assert _class in self.structures, 'Class should be one of %s; you gave %s' % (self.structures, _class)
+ self.code = _class
+ self.info = self.structures[_class]()
+
+
+QueryInformationJobObjectProto = WINFUNCTYPE(
+ BOOL, # Return type
+ HANDLE, # hJob
+ DWORD, # JobObjectInfoClass
+ LPVOID, # lpJobObjectInfo
+ DWORD, # cbJobObjectInfoLength
+ LPDWORD # lpReturnLength
+ )
+
+QueryInformationJobObjectFlags = (
+ (1, 'hJob'),
+ (1, 'JobObjectInfoClass'),
+ (1, 'lpJobObjectInfo'),
+ (1, 'cbJobObjectInfoLength'),
+ (1, 'lpReturnLength', None)
+ )
+
+_QueryInformationJobObject = QueryInformationJobObjectProto(
+ ('QueryInformationJobObject', windll.kernel32),
+ QueryInformationJobObjectFlags
+ )
+
+class SubscriptableReadOnlyStruct(object):
+ def __init__(self, struct):
+ self._struct = struct
+
+ def _delegate(self, name):
+ result = getattr(self._struct, name)
+ if isinstance(result, Structure):
+ return SubscriptableReadOnlyStruct(result)
+ return result
+
+ def __getitem__(self, name):
+ match = [fname for fname, ftype in self._struct._fields_
+ if fname == name]
+ if match:
+ return self._delegate(name)
+ raise KeyError(name)
+
+ def __getattr__(self, name):
+ return self._delegate(name)
+
+def QueryInformationJobObject(hJob, JobObjectInfoClass):
+ jobinfo = JobObjectInfo(JobObjectInfoClass)
+ result = _QueryInformationJobObject(
+ hJob=hJob,
+ JobObjectInfoClass=jobinfo.code,
+ lpJobObjectInfo=addressof(jobinfo.info),
+ cbJobObjectInfoLength=sizeof(jobinfo.info)
+ )
+ if not result:
+ raise WinError()
+ return SubscriptableReadOnlyStruct(jobinfo.info)
+
+def test_qijo():
+ from killableprocess import Popen
+
+ popen = Popen('c:\\windows\\notepad.exe')
+
+ try:
+ result = QueryInformationJobObject(0, 8)
+ raise AssertionError('throw should occur')
+ except WindowsError, e:
+ pass
+
+ try:
+ result = QueryInformationJobObject(0, 1)
+ raise AssertionError('throw should occur')
+ except NotImplementedError, e:
+ pass
+
+ result = QueryInformationJobObject(popen._job, 8)
+ if result['BasicInfo']['ActiveProcesses'] != 1:
+ raise AssertionError('expected ActiveProcesses to be 1')
+ popen.kill()
+
+ result = QueryInformationJobObject(popen._job, 8)
+ if result.BasicInfo.ActiveProcesses != 0:
+ raise AssertionError('expected ActiveProcesses to be 0')
+
+if __name__ == '__main__':
+ print "testing."
+ test_qijo()
+ print "success!"
diff --git a/mozrunner/winprocess.py b/mozrunner/winprocess.py
new file mode 100644
index 0000000..08f7fdd
--- /dev/null
+++ b/mozrunner/winprocess.py
@@ -0,0 +1,381 @@
+# A module to expose various thread/process/job related structures and
+# methods from kernel32
+#
+# The MIT License
+#
+# Copyright (c) 2003-2004 by Peter Astrand <astrand@lysator.liu.se>
+#
+# Additions and modifications written by Benjamin Smedberg
+# <benjamin@smedbergs.us> are Copyright (c) 2006 by the Mozilla Foundation
+# <http://www.mozilla.org/>
+#
+# More Modifications
+# Copyright (c) 2006-2007 by Mike Taylor <bear@code-bear.com>
+# Copyright (c) 2007-2008 by Mikeal Rogers <mikeal@mozilla.com>
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of the
+# author not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
+# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from ctypes import c_void_p, POINTER, sizeof, Structure, windll, WinError, WINFUNCTYPE
+from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LPCWSTR, LPWSTR, UINT, WORD
+from qijo import QueryInformationJobObject
+
+LPVOID = c_void_p
+LPBYTE = POINTER(BYTE)
+LPDWORD = POINTER(DWORD)
+LPBOOL = POINTER(BOOL)
+
+def ErrCheckBool(result, func, args):
+ """errcheck function for Windows functions that return a BOOL True
+ on success"""
+ if not result:
+ raise WinError()
+ return args
+
+
+# AutoHANDLE
+
+class AutoHANDLE(HANDLE):
+ """Subclass of HANDLE which will call CloseHandle() on deletion."""
+
+ CloseHandleProto = WINFUNCTYPE(BOOL, HANDLE)
+ CloseHandle = CloseHandleProto(("CloseHandle", windll.kernel32))
+ CloseHandle.errcheck = ErrCheckBool
+
+ def Close(self):
+ if self.value and self.value != HANDLE(-1).value:
+ self.CloseHandle(self)
+ self.value = 0
+
+ def __del__(self):
+ self.Close()
+
+ def __int__(self):
+ return self.value
+
+def ErrCheckHandle(result, func, args):
+ """errcheck function for Windows functions that return a HANDLE."""
+ if not result:
+ raise WinError()
+ return AutoHANDLE(result)
+
+# PROCESS_INFORMATION structure
+
+class PROCESS_INFORMATION(Structure):
+ _fields_ = [("hProcess", HANDLE),
+ ("hThread", HANDLE),
+ ("dwProcessID", DWORD),
+ ("dwThreadID", DWORD)]
+
+ def __init__(self):
+ Structure.__init__(self)
+
+ self.cb = sizeof(self)
+
+LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)
+
+# STARTUPINFO structure
+
+class STARTUPINFO(Structure):
+ _fields_ = [("cb", DWORD),
+ ("lpReserved", LPWSTR),
+ ("lpDesktop", LPWSTR),
+ ("lpTitle", LPWSTR),
+ ("dwX", DWORD),
+ ("dwY", DWORD),
+ ("dwXSize", DWORD),
+ ("dwYSize", DWORD),
+ ("dwXCountChars", DWORD),
+ ("dwYCountChars", DWORD),
+ ("dwFillAttribute", DWORD),
+ ("dwFlags", DWORD),
+ ("wShowWindow", WORD),
+ ("cbReserved2", WORD),
+ ("lpReserved2", LPBYTE),
+ ("hStdInput", HANDLE),
+ ("hStdOutput", HANDLE),
+ ("hStdError", HANDLE)
+ ]
+LPSTARTUPINFO = POINTER(STARTUPINFO)
+
+SW_HIDE = 0
+
+STARTF_USESHOWWINDOW = 0x01
+STARTF_USESIZE = 0x02
+STARTF_USEPOSITION = 0x04
+STARTF_USECOUNTCHARS = 0x08
+STARTF_USEFILLATTRIBUTE = 0x10
+STARTF_RUNFULLSCREEN = 0x20
+STARTF_FORCEONFEEDBACK = 0x40
+STARTF_FORCEOFFFEEDBACK = 0x80
+STARTF_USESTDHANDLES = 0x100
+
+# EnvironmentBlock
+
+class EnvironmentBlock:
+ """An object which can be passed as the lpEnv parameter of CreateProcess.
+ It is initialized with a dictionary."""
+
+ def __init__(self, dict):
+ if not dict:
+ self._as_parameter_ = None
+ else:
+ values = ["%s=%s" % (key, value)
+ for (key, value) in dict.iteritems()]
+ values.append("")
+ self._as_parameter_ = LPCWSTR("\0".join(values))
+
+# CreateProcess()
+
+CreateProcessProto = WINFUNCTYPE(BOOL, # Return type
+ LPCWSTR, # lpApplicationName
+ LPWSTR, # lpCommandLine
+ LPVOID, # lpProcessAttributes
+ LPVOID, # lpThreadAttributes
+ BOOL, # bInheritHandles
+ DWORD, # dwCreationFlags
+ LPVOID, # lpEnvironment
+ LPCWSTR, # lpCurrentDirectory
+ LPSTARTUPINFO, # lpStartupInfo
+ LPPROCESS_INFORMATION # lpProcessInformation
+ )
+
+CreateProcessFlags = ((1, "lpApplicationName", None),
+ (1, "lpCommandLine"),
+ (1, "lpProcessAttributes", None),
+ (1, "lpThreadAttributes", None),
+ (1, "bInheritHandles", True),
+ (1, "dwCreationFlags", 0),
+ (1, "lpEnvironment", None),
+ (1, "lpCurrentDirectory", None),
+ (1, "lpStartupInfo"),
+ (2, "lpProcessInformation"))
+
+def ErrCheckCreateProcess(result, func, args):
+ ErrCheckBool(result, func, args)
+ # return a tuple (hProcess, hThread, dwProcessID, dwThreadID)
+ pi = args[9]
+ return AutoHANDLE(pi.hProcess), AutoHANDLE(pi.hThread), pi.dwProcessID, pi.dwThreadID
+
+CreateProcess = CreateProcessProto(("CreateProcessW", windll.kernel32),
+ CreateProcessFlags)
+CreateProcess.errcheck = ErrCheckCreateProcess
+
+# flags for CreateProcess
+CREATE_BREAKAWAY_FROM_JOB = 0x01000000
+CREATE_DEFAULT_ERROR_MODE = 0x04000000
+CREATE_NEW_CONSOLE = 0x00000010
+CREATE_NEW_PROCESS_GROUP = 0x00000200
+CREATE_NO_WINDOW = 0x08000000
+CREATE_SUSPENDED = 0x00000004
+CREATE_UNICODE_ENVIRONMENT = 0x00000400
+
+# flags for job limit information
+# see http://msdn.microsoft.com/en-us/library/ms684147%28VS.85%29.aspx
+JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x00000800
+JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK = 0x00001000
+
+# XXX these flags should be documented
+DEBUG_ONLY_THIS_PROCESS = 0x00000002
+DEBUG_PROCESS = 0x00000001
+DETACHED_PROCESS = 0x00000008
+
+# CreateJobObject()
+
+CreateJobObjectProto = WINFUNCTYPE(HANDLE, # Return type
+ LPVOID, # lpJobAttributes
+ LPCWSTR # lpName
+ )
+
+CreateJobObjectFlags = ((1, "lpJobAttributes", None),
+ (1, "lpName", None))
+
+CreateJobObject = CreateJobObjectProto(("CreateJobObjectW", windll.kernel32),
+ CreateJobObjectFlags)
+CreateJobObject.errcheck = ErrCheckHandle
+
+# AssignProcessToJobObject()
+
+AssignProcessToJobObjectProto = WINFUNCTYPE(BOOL, # Return type
+ HANDLE, # hJob
+ HANDLE # hProcess
+ )
+AssignProcessToJobObjectFlags = ((1, "hJob"),
+ (1, "hProcess"))
+AssignProcessToJobObject = AssignProcessToJobObjectProto(
+ ("AssignProcessToJobObject", windll.kernel32),
+ AssignProcessToJobObjectFlags)
+AssignProcessToJobObject.errcheck = ErrCheckBool
+
+# GetCurrentProcess()
+# because os.getPid() is way too easy
+GetCurrentProcessProto = WINFUNCTYPE(HANDLE # Return type
+ )
+GetCurrentProcessFlags = ()
+GetCurrentProcess = GetCurrentProcessProto(
+ ("GetCurrentProcess", windll.kernel32),
+ GetCurrentProcessFlags)
+GetCurrentProcess.errcheck = ErrCheckHandle
+
+# IsProcessInJob()
+try:
+ IsProcessInJobProto = WINFUNCTYPE(BOOL, # Return type
+ HANDLE, # Process Handle
+ HANDLE, # Job Handle
+ LPBOOL # Result
+ )
+ IsProcessInJobFlags = ((1, "ProcessHandle"),
+ (1, "JobHandle", HANDLE(0)),
+ (2, "Result"))
+ IsProcessInJob = IsProcessInJobProto(
+ ("IsProcessInJob", windll.kernel32),
+ IsProcessInJobFlags)
+ IsProcessInJob.errcheck = ErrCheckBool
+except AttributeError:
+ # windows 2k doesn't have this API
+ def IsProcessInJob(process):
+ return False
+
+
+# ResumeThread()
+
+def ErrCheckResumeThread(result, func, args):
+ if result == -1:
+ raise WinError()
+
+ return args
+
+ResumeThreadProto = WINFUNCTYPE(DWORD, # Return type
+ HANDLE # hThread
+ )
+ResumeThreadFlags = ((1, "hThread"),)
+ResumeThread = ResumeThreadProto(("ResumeThread", windll.kernel32),
+ ResumeThreadFlags)
+ResumeThread.errcheck = ErrCheckResumeThread
+
+# TerminateProcess()
+
+TerminateProcessProto = WINFUNCTYPE(BOOL, # Return type
+ HANDLE, # hProcess
+ UINT # uExitCode
+ )
+TerminateProcessFlags = ((1, "hProcess"),
+ (1, "uExitCode", 127))
+TerminateProcess = TerminateProcessProto(
+ ("TerminateProcess", windll.kernel32),
+ TerminateProcessFlags)
+TerminateProcess.errcheck = ErrCheckBool
+
+# TerminateJobObject()
+
+TerminateJobObjectProto = WINFUNCTYPE(BOOL, # Return type
+ HANDLE, # hJob
+ UINT # uExitCode
+ )
+TerminateJobObjectFlags = ((1, "hJob"),
+ (1, "uExitCode", 127))
+TerminateJobObject = TerminateJobObjectProto(
+ ("TerminateJobObject", windll.kernel32),
+ TerminateJobObjectFlags)
+TerminateJobObject.errcheck = ErrCheckBool
+
+# WaitForSingleObject()
+
+WaitForSingleObjectProto = WINFUNCTYPE(DWORD, # Return type
+ HANDLE, # hHandle
+ DWORD, # dwMilliseconds
+ )
+WaitForSingleObjectFlags = ((1, "hHandle"),
+ (1, "dwMilliseconds", -1))
+WaitForSingleObject = WaitForSingleObjectProto(
+ ("WaitForSingleObject", windll.kernel32),
+ WaitForSingleObjectFlags)
+
+INFINITE = -1
+WAIT_TIMEOUT = 0x0102
+WAIT_OBJECT_0 = 0x0
+WAIT_ABANDONED = 0x0080
+
+# GetExitCodeProcess()
+
+GetExitCodeProcessProto = WINFUNCTYPE(BOOL, # Return type
+ HANDLE, # hProcess
+ LPDWORD, # lpExitCode
+ )
+GetExitCodeProcessFlags = ((1, "hProcess"),
+ (2, "lpExitCode"))
+GetExitCodeProcess = GetExitCodeProcessProto(
+ ("GetExitCodeProcess", windll.kernel32),
+ GetExitCodeProcessFlags)
+GetExitCodeProcess.errcheck = ErrCheckBool
+
+def CanCreateJobObject():
+ currentProc = GetCurrentProcess()
+ if IsProcessInJob(currentProc):
+ jobinfo = QueryInformationJobObject(HANDLE(0), 'JobObjectExtendedLimitInformation')
+ limitflags = jobinfo['BasicLimitInformation']['LimitFlags']
+ return bool(limitflags & JOB_OBJECT_LIMIT_BREAKAWAY_OK) or bool(limitflags & JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK)
+ else:
+ return True
+
+### testing functions
+
+def parent():
+ print 'Starting parent'
+ currentProc = GetCurrentProcess()
+ if IsProcessInJob(currentProc):
+ print >> sys.stderr, "You should not be in a job object to test"
+ sys.exit(1)
+ assert CanCreateJobObject()
+ print 'File: %s' % __file__
+ command = [sys.executable, __file__, '-child']
+ print 'Running command: %s' % command
+ process = Popen(command)
+ process.kill()
+ code = process.returncode
+ print 'Child code: %s' % code
+ assert code == 127
+
+def child():
+ print 'Starting child'
+ currentProc = GetCurrentProcess()
+ injob = IsProcessInJob(currentProc)
+ print "Is in a job?: %s" % injob
+ can_create = CanCreateJobObject()
+ print 'Can create job?: %s' % can_create
+ process = Popen('c:\\windows\\notepad.exe')
+ assert process._job
+ jobinfo = QueryInformationJobObject(process._job, 'JobObjectExtendedLimitInformation')
+ print 'Job info: %s' % jobinfo
+ limitflags = jobinfo['BasicLimitInformation']['LimitFlags']
+ print 'LimitFlags: %s' % limitflags
+ process.kill()
+
+if __name__ == '__main__':
+ import sys
+ from killableprocess import Popen
+ nargs = len(sys.argv[1:])
+ if nargs:
+ if nargs != 1 or sys.argv[1] != '-child':
+ raise AssertionError('Wrong flags; run like `python /path/to/winprocess.py`')
+ child()
+ else:
+ parent()
diff --git a/mozrunner/wpk.py b/mozrunner/wpk.py
new file mode 100644
index 0000000..46993cc
--- /dev/null
+++ b/mozrunner/wpk.py
@@ -0,0 +1,81 @@
+from ctypes import sizeof, windll, addressof, c_wchar, create_unicode_buffer
+from ctypes.wintypes import DWORD, HANDLE
+
+PROCESS_TERMINATE = 0x0001
+PROCESS_QUERY_INFORMATION = 0x0400
+PROCESS_VM_READ = 0x0010
+
+def get_pids(process_name):
+ BIG_ARRAY = DWORD * 4096
+ processes = BIG_ARRAY()
+ needed = DWORD()
+
+ pids = []
+ result = windll.psapi.EnumProcesses(processes,
+ sizeof(processes),
+ addressof(needed))
+ if not result:
+ return pids
+
+ num_results = needed.value / sizeof(DWORD)
+
+ for i in range(num_results):
+ pid = processes[i]
+ process = windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION |
+ PROCESS_VM_READ,
+ 0, pid)
+ if process:
+ module = HANDLE()
+ result = windll.psapi.EnumProcessModules(process,
+ addressof(module),
+ sizeof(module),
+ addressof(needed))
+ if result:
+ name = create_unicode_buffer(1024)
+ result = windll.psapi.GetModuleBaseNameW(process, module,
+ name, len(name))
+ # TODO: This might not be the best way to
+ # match a process name; maybe use a regexp instead.
+ if name.value.startswith(process_name):
+ pids.append(pid)
+ windll.kernel32.CloseHandle(module)
+ windll.kernel32.CloseHandle(process)
+
+ return pids
+
+def kill_pid(pid):
+ process = windll.kernel32.OpenProcess(PROCESS_TERMINATE, 0, pid)
+ if process:
+ windll.kernel32.TerminateProcess(process, 0)
+ windll.kernel32.CloseHandle(process)
+
+def kill_process_by_name(name):
+ pids = get_pids(name)
+ for pid in pids:
+ kill_pid(pid)
+
+if __name__ == '__main__':
+ import subprocess
+ import time
+
+ # This test just opens a new notepad instance and kills it.
+
+ name = 'notepad'
+
+ old_pids = set(get_pids(name))
+ subprocess.Popen([name])
+ time.sleep(0.25)
+ new_pids = set(get_pids(name)).difference(old_pids)
+
+ if len(new_pids) != 1:
+ raise Exception('%s was not opened or get_pids() is '
+ 'malfunctioning' % name)
+
+ kill_pid(tuple(new_pids)[0])
+
+ newest_pids = set(get_pids(name)).difference(old_pids)
+
+ if len(newest_pids) != 0:
+ raise Exception('kill_pid() is malfunctioning')
+
+ print "Test passed."
diff --git a/simplejson/LICENSE b/simplejson/LICENSE
new file mode 100644
index 0000000..ad95f29
--- /dev/null
+++ b/simplejson/LICENSE
@@ -0,0 +1,19 @@
+Copyright (c) 2006 Bob Ippolito
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/simplejson/README b/simplejson/README
new file mode 100644
index 0000000..69a7bd2
--- /dev/null
+++ b/simplejson/README
@@ -0,0 +1,11 @@
+URL: http://pypi.python.org/packages/source/s/simplejson/simplejson-2.1.2.tar.gz#md5=a856f9ae9ab3749991a93ddeafadc554
+Version: 2.1.2
+License: MIT
+License File: LICENSE
+
+Description:
+Externally maintained JSON module. Needed for compatibility with Python before 2.6.
+
+
+Local Modifications:
+No Modifications
diff --git a/simplejson/__init__.py b/simplejson/__init__.py
new file mode 100644
index 0000000..3857129
--- /dev/null
+++ b/simplejson/__init__.py
@@ -0,0 +1,437 @@
+r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of
+JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
+interchange format.
+
+:mod:`simplejson` exposes an API familiar to users of the standard library
+:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained
+version of the :mod:`json` library contained in Python 2.6, but maintains
+compatibility with Python 2.4 and Python 2.5 and (currently) has
+significant performance advantages, even without using the optional C
+extension for speedups.
+
+Encoding basic Python object hierarchies::
+
+ >>> import simplejson as json
+ >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
+ '["foo", {"bar": ["baz", null, 1.0, 2]}]'
+ >>> print json.dumps("\"foo\bar")
+ "\"foo\bar"
+ >>> print json.dumps(u'\u1234')
+ "\u1234"
+ >>> print json.dumps('\\')
+ "\\"
+ >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)
+ {"a": 0, "b": 0, "c": 0}
+ >>> from StringIO import StringIO
+ >>> io = StringIO()
+ >>> json.dump(['streaming API'], io)
+ >>> io.getvalue()
+ '["streaming API"]'
+
+Compact encoding::
+
+ >>> import simplejson as json
+ >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':'))
+ '[1,2,3,{"4":5,"6":7}]'
+
+Pretty printing::
+
+ >>> import simplejson as json
+ >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ')
+ >>> print '\n'.join([l.rstrip() for l in s.splitlines()])
+ {
+ "4": 5,
+ "6": 7
+ }
+
+Decoding JSON::
+
+ >>> import simplejson as json
+ >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}]
+ >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj
+ True
+ >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar'
+ True
+ >>> from StringIO import StringIO
+ >>> io = StringIO('["streaming API"]')
+ >>> json.load(io)[0] == 'streaming API'
+ True
+
+Specializing JSON object decoding::
+
+ >>> import simplejson as json
+ >>> def as_complex(dct):
+ ... if '__complex__' in dct:
+ ... return complex(dct['real'], dct['imag'])
+ ... return dct
+ ...
+ >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}',
+ ... object_hook=as_complex)
+ (1+2j)
+ >>> from decimal import Decimal
+ >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1')
+ True
+
+Specializing JSON object encoding::
+
+ >>> import simplejson as json
+ >>> def encode_complex(obj):
+ ... if isinstance(obj, complex):
+ ... return [obj.real, obj.imag]
+ ... raise TypeError(repr(o) + " is not JSON serializable")
+ ...
+ >>> json.dumps(2 + 1j, default=encode_complex)
+ '[2.0, 1.0]'
+ >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j)
+ '[2.0, 1.0]'
+ >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j))
+ '[2.0, 1.0]'
+
+
+Using simplejson.tool from the shell to validate and pretty-print::
+
+ $ echo '{"json":"obj"}' | python -m simplejson.tool
+ {
+ "json": "obj"
+ }
+ $ echo '{ 1.2:3.4}' | python -m simplejson.tool
+ Expecting property name: line 1 column 2 (char 2)
+"""
+__version__ = '2.1.2'
+__all__ = [
+ 'dump', 'dumps', 'load', 'loads',
+ 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder',
+ 'OrderedDict',
+]
+
+__author__ = 'Bob Ippolito <bob@redivi.com>'
+
+from decimal import Decimal
+
+from decoder import JSONDecoder, JSONDecodeError
+from encoder import JSONEncoder
+def _import_OrderedDict():
+ import collections
+ try:
+ return collections.OrderedDict
+ except AttributeError:
+ import ordered_dict
+ return ordered_dict.OrderedDict
+OrderedDict = _import_OrderedDict()
+
+def _import_c_make_encoder():
+ try:
+ from simplejson._speedups import make_encoder
+ return make_encoder
+ except ImportError:
+ return None
+
+_default_encoder = JSONEncoder(
+ skipkeys=False,
+ ensure_ascii=True,
+ check_circular=True,
+ allow_nan=True,
+ indent=None,
+ separators=None,
+ encoding='utf-8',
+ default=None,
+ use_decimal=False,
+)
+
+def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
+ allow_nan=True, cls=None, indent=None, separators=None,
+ encoding='utf-8', default=None, use_decimal=False, **kw):
+ """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
+ ``.write()``-supporting file-like object).
+
+ If ``skipkeys`` is true then ``dict`` keys that are not basic types
+ (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
+ will be skipped instead of raising a ``TypeError``.
+
+ If ``ensure_ascii`` is false, then the some chunks written to ``fp``
+ may be ``unicode`` instances, subject to normal Python ``str`` to
+ ``unicode`` coercion rules. Unless ``fp.write()`` explicitly
+ understands ``unicode`` (as in ``codecs.getwriter()``) this is likely
+ to cause an error.
+
+ If ``check_circular`` is false, then the circular reference check
+ for container types will be skipped and a circular reference will
+ result in an ``OverflowError`` (or worse).
+
+ If ``allow_nan`` is false, then it will be a ``ValueError`` to
+ serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
+ in strict compliance of the JSON specification, instead of using the
+ JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
+
+ If *indent* is a string, then JSON array elements and object members
+ will be pretty-printed with a newline followed by that string repeated
+ for each level of nesting. ``None`` (the default) selects the most compact
+ representation without any newlines. For backwards compatibility with
+ versions of simplejson earlier than 2.1.0, an integer is also accepted
+ and is converted to a string with that many spaces.
+
+ If ``separators`` is an ``(item_separator, dict_separator)`` tuple
+ then it will be used instead of the default ``(', ', ': ')`` separators.
+ ``(',', ':')`` is the most compact JSON representation.
+
+ ``encoding`` is the character encoding for str instances, default is UTF-8.
+
+ ``default(obj)`` is a function that should return a serializable version
+ of obj or raise TypeError. The default simply raises TypeError.
+
+ If *use_decimal* is true (default: ``False``) then decimal.Decimal
+ will be natively serialized to JSON with full precision.
+
+ To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
+ ``.default()`` method to serialize additional types), specify it with
+ the ``cls`` kwarg.
+
+ """
+ # cached encoder
+ if (not skipkeys and ensure_ascii and
+ check_circular and allow_nan and
+ cls is None and indent is None and separators is None and
+ encoding == 'utf-8' and default is None and not kw):
+ iterable = _default_encoder.iterencode(obj)
+ else:
+ if cls is None:
+ cls = JSONEncoder
+ iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
+ check_circular=check_circular, allow_nan=allow_nan, indent=indent,
+ separators=separators, encoding=encoding,
+ default=default, use_decimal=use_decimal, **kw).iterencode(obj)
+ # could accelerate with writelines in some versions of Python, at
+ # a debuggability cost
+ for chunk in iterable:
+ fp.write(chunk)
+
+
+def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
+ allow_nan=True, cls=None, indent=None, separators=None,
+ encoding='utf-8', default=None, use_decimal=False, **kw):
+ """Serialize ``obj`` to a JSON formatted ``str``.
+
+ If ``skipkeys`` is false then ``dict`` keys that are not basic types
+ (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
+ will be skipped instead of raising a ``TypeError``.
+
+ If ``ensure_ascii`` is false, then the return value will be a
+ ``unicode`` instance subject to normal Python ``str`` to ``unicode``
+ coercion rules instead of being escaped to an ASCII ``str``.
+
+ If ``check_circular`` is false, then the circular reference check
+ for container types will be skipped and a circular reference will
+ result in an ``OverflowError`` (or worse).
+
+ If ``allow_nan`` is false, then it will be a ``ValueError`` to
+ serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
+ strict compliance of the JSON specification, instead of using the
+ JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
+
+ If ``indent`` is a string, then JSON array elements and object members
+ will be pretty-printed with a newline followed by that string repeated
+ for each level of nesting. ``None`` (the default) selects the most compact
+ representation without any newlines. For backwards compatibility with
+ versions of simplejson earlier than 2.1.0, an integer is also accepted
+ and is converted to a string with that many spaces.
+
+ If ``separators`` is an ``(item_separator, dict_separator)`` tuple
+ then it will be used instead of the default ``(', ', ': ')`` separators.
+ ``(',', ':')`` is the most compact JSON representation.
+
+ ``encoding`` is the character encoding for str instances, default is UTF-8.
+
+ ``default(obj)`` is a function that should return a serializable version
+ of obj or raise TypeError. The default simply raises TypeError.
+
+ If *use_decimal* is true (default: ``False``) then decimal.Decimal
+ will be natively serialized to JSON with full precision.
+
+ To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
+ ``.default()`` method to serialize additional types), specify it with
+ the ``cls`` kwarg.
+
+ """
+ # cached encoder
+ if (not skipkeys and ensure_ascii and
+ check_circular and allow_nan and
+ cls is None and indent is None and separators is None and
+ encoding == 'utf-8' and default is None and not use_decimal
+ and not kw):
+ return _default_encoder.encode(obj)
+ if cls is None:
+ cls = JSONEncoder
+ return cls(
+ skipkeys=skipkeys, ensure_ascii=ensure_ascii,
+ check_circular=check_circular, allow_nan=allow_nan, indent=indent,
+ separators=separators, encoding=encoding, default=default,
+ use_decimal=use_decimal, **kw).encode(obj)
+
+
+_default_decoder = JSONDecoder(encoding=None, object_hook=None,
+ object_pairs_hook=None)
+
+
+def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None,
+ parse_int=None, parse_constant=None, object_pairs_hook=None,
+ use_decimal=False, **kw):
+ """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
+ a JSON document) to a Python object.
+
+ *encoding* determines the encoding used to interpret any
+ :class:`str` objects decoded by this instance (``'utf-8'`` by
+ default). It has no effect when decoding :class:`unicode` objects.
+
+ Note that currently only encodings that are a superset of ASCII work,
+ strings of other encodings should be passed in as :class:`unicode`.
+
+ *object_hook*, if specified, will be called with the result of every
+ JSON object decoded and its return value will be used in place of the
+ given :class:`dict`. This can be used to provide custom
+ deserializations (e.g. to support JSON-RPC class hinting).
+
+ *object_pairs_hook* is an optional function that will be called with
+ the result of any object literal decode with an ordered list of pairs.
+ The return value of *object_pairs_hook* will be used instead of the
+ :class:`dict`. This feature can be used to implement custom decoders
+ that rely on the order that the key and value pairs are decoded (for
+ example, :func:`collections.OrderedDict` will remember the order of
+ insertion). If *object_hook* is also defined, the *object_pairs_hook*
+ takes priority.
+
+ *parse_float*, if specified, will be called with the string of every
+ JSON float to be decoded. By default, this is equivalent to
+ ``float(num_str)``. This can be used to use another datatype or parser
+ for JSON floats (e.g. :class:`decimal.Decimal`).
+
+ *parse_int*, if specified, will be called with the string of every
+ JSON int to be decoded. By default, this is equivalent to
+ ``int(num_str)``. This can be used to use another datatype or parser
+ for JSON integers (e.g. :class:`float`).
+
+ *parse_constant*, if specified, will be called with one of the
+ following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This
+ can be used to raise an exception if invalid JSON numbers are
+ encountered.
+
+ If *use_decimal* is true (default: ``False``) then it implies
+ parse_float=decimal.Decimal for parity with ``dump``.
+
+ To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
+ kwarg.
+
+ """
+ return loads(fp.read(),
+ encoding=encoding, cls=cls, object_hook=object_hook,
+ parse_float=parse_float, parse_int=parse_int,
+ parse_constant=parse_constant, object_pairs_hook=object_pairs_hook,
+ use_decimal=use_decimal, **kw)
+
+
+def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None,
+ parse_int=None, parse_constant=None, object_pairs_hook=None,
+ use_decimal=False, **kw):
+ """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON
+ document) to a Python object.
+
+ *encoding* determines the encoding used to interpret any
+ :class:`str` objects decoded by this instance (``'utf-8'`` by
+ default). It has no effect when decoding :class:`unicode` objects.
+
+ Note that currently only encodings that are a superset of ASCII work,
+ strings of other encodings should be passed in as :class:`unicode`.
+
+ *object_hook*, if specified, will be called with the result of every
+ JSON object decoded and its return value will be used in place of the
+ given :class:`dict`. This can be used to provide custom
+ deserializations (e.g. to support JSON-RPC class hinting).
+
+ *object_pairs_hook* is an optional function that will be called with
+ the result of any object literal decode with an ordered list of pairs.
+ The return value of *object_pairs_hook* will be used instead of the
+ :class:`dict`. This feature can be used to implement custom decoders
+ that rely on the order that the key and value pairs are decoded (for
+ example, :func:`collections.OrderedDict` will remember the order of
+ insertion). If *object_hook* is also defined, the *object_pairs_hook*
+ takes priority.
+
+ *parse_float*, if specified, will be called with the string of every
+ JSON float to be decoded. By default, this is equivalent to
+ ``float(num_str)``. This can be used to use another datatype or parser
+ for JSON floats (e.g. :class:`decimal.Decimal`).
+
+ *parse_int*, if specified, will be called with the string of every
+ JSON int to be decoded. By default, this is equivalent to
+ ``int(num_str)``. This can be used to use another datatype or parser
+ for JSON integers (e.g. :class:`float`).
+
+ *parse_constant*, if specified, will be called with one of the
+ following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This
+ can be used to raise an exception if invalid JSON numbers are
+ encountered.
+
+ If *use_decimal* is true (default: ``False``) then it implies
+ parse_float=decimal.Decimal for parity with ``dump``.
+
+ To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
+ kwarg.
+
+ """
+ if (cls is None and encoding is None and object_hook is None and
+ parse_int is None and parse_float is None and
+ parse_constant is None and object_pairs_hook is None
+ and not use_decimal and not kw):
+ return _default_decoder.decode(s)
+ if cls is None:
+ cls = JSONDecoder
+ if object_hook is not None:
+ kw['object_hook'] = object_hook
+ if object_pairs_hook is not None:
+ kw['object_pairs_hook'] = object_pairs_hook
+ if parse_float is not None:
+ kw['parse_float'] = parse_float
+ if parse_int is not None:
+ kw['parse_int'] = parse_int
+ if parse_constant is not None:
+ kw['parse_constant'] = parse_constant
+ if use_decimal:
+ if parse_float is not None:
+ raise TypeError("use_decimal=True implies parse_float=Decimal")
+ kw['parse_float'] = Decimal
+ return cls(encoding=encoding, **kw).decode(s)
+
+
+def _toggle_speedups(enabled):
+ import simplejson.decoder as dec
+ import simplejson.encoder as enc
+ import simplejson.scanner as scan
+ c_make_encoder = _import_c_make_encoder()
+ if enabled:
+ dec.scanstring = dec.c_scanstring or dec.py_scanstring
+ enc.c_make_encoder = c_make_encoder
+ enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or
+ enc.py_encode_basestring_ascii)
+ scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner
+ else:
+ dec.scanstring = dec.py_scanstring
+ enc.c_make_encoder = None
+ enc.encode_basestring_ascii = enc.py_encode_basestring_ascii
+ scan.make_scanner = scan.py_make_scanner
+ dec.make_scanner = scan.make_scanner
+ global _default_decoder
+ _default_decoder = JSONDecoder(
+ encoding=None,
+ object_hook=None,
+ object_pairs_hook=None,
+ )
+ global _default_encoder
+ _default_encoder = JSONEncoder(
+ skipkeys=False,
+ ensure_ascii=True,
+ check_circular=True,
+ allow_nan=True,
+ indent=None,
+ separators=None,
+ encoding='utf-8',
+ default=None,
+ )
diff --git a/simplejson/decoder.py b/simplejson/decoder.py
new file mode 100644
index 0000000..e5496d6
--- /dev/null
+++ b/simplejson/decoder.py
@@ -0,0 +1,421 @@
+"""Implementation of JSONDecoder
+"""
+import re
+import sys
+import struct
+
+from simplejson.scanner import make_scanner
+def _import_c_scanstring():
+ try:
+ from simplejson._speedups import scanstring
+ return scanstring
+ except ImportError:
+ return None
+c_scanstring = _import_c_scanstring()
+
+__all__ = ['JSONDecoder']
+
+FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
+
+def _floatconstants():
+ _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
+ # The struct module in Python 2.4 would get frexp() out of range here
+ # when an endian is specified in the format string. Fixed in Python 2.5+
+ if sys.byteorder != 'big':
+ _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
+ nan, inf = struct.unpack('dd', _BYTES)
+ return nan, inf, -inf
+
+NaN, PosInf, NegInf = _floatconstants()
+
+
+class JSONDecodeError(ValueError):
+ """Subclass of ValueError with the following additional properties:
+
+ msg: The unformatted error message
+ doc: The JSON document being parsed
+ pos: The start index of doc where parsing failed
+ end: The end index of doc where parsing failed (may be None)
+ lineno: The line corresponding to pos
+ colno: The column corresponding to pos
+ endlineno: The line corresponding to end (may be None)
+ endcolno: The column corresponding to end (may be None)
+
+ """
+ def __init__(self, msg, doc, pos, end=None):
+ ValueError.__init__(self, errmsg(msg, doc, pos, end=end))
+ self.msg = msg
+ self.doc = doc
+ self.pos = pos
+ self.end = end
+ self.lineno, self.colno = linecol(doc, pos)
+ if end is not None:
+ self.endlineno, self.endcolno = linecol(doc, end)
+ else:
+ self.endlineno, self.endcolno = None, None
+
+
+def linecol(doc, pos):
+ lineno = doc.count('\n', 0, pos) + 1
+ if lineno == 1:
+ colno = pos
+ else:
+ colno = pos - doc.rindex('\n', 0, pos)
+ return lineno, colno
+
+
+def errmsg(msg, doc, pos, end=None):
+ # Note that this function is called from _speedups
+ lineno, colno = linecol(doc, pos)
+ if end is None:
+ #fmt = '{0}: line {1} column {2} (char {3})'
+ #return fmt.format(msg, lineno, colno, pos)
+ fmt = '%s: line %d column %d (char %d)'
+ return fmt % (msg, lineno, colno, pos)
+ endlineno, endcolno = linecol(doc, end)
+ #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
+ #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
+ fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
+ return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
+
+
+_CONSTANTS = {
+ '-Infinity': NegInf,
+ 'Infinity': PosInf,
+ 'NaN': NaN,
+}
+
+STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
+BACKSLASH = {
+ '"': u'"', '\\': u'\\', '/': u'/',
+ 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
+}
+
+DEFAULT_ENCODING = "utf-8"
+
+def py_scanstring(s, end, encoding=None, strict=True,
+ _b=BACKSLASH, _m=STRINGCHUNK.match):
+ """Scan the string s for a JSON string. End is the index of the
+ character in s after the quote that started the JSON string.
+ Unescapes all valid JSON string escape sequences and raises ValueError
+ on attempt to decode an invalid string. If strict is False then literal
+ control characters are allowed in the string.
+
+ Returns a tuple of the decoded string and the index of the character in s
+ after the end quote."""
+ if encoding is None:
+ encoding = DEFAULT_ENCODING
+ chunks = []
+ _append = chunks.append
+ begin = end - 1
+ while 1:
+ chunk = _m(s, end)
+ if chunk is None:
+ raise JSONDecodeError(
+ "Unterminated string starting at", s, begin)
+ end = chunk.end()
+ content, terminator = chunk.groups()
+ # Content is contains zero or more unescaped string characters
+ if content:
+ if not isinstance(content, unicode):
+ content = unicode(content, encoding)
+ _append(content)
+ # Terminator is the end of string, a literal control character,
+ # or a backslash denoting that an escape sequence follows
+ if terminator == '"':
+ break
+ elif terminator != '\\':
+ if strict:
+ msg = "Invalid control character %r at" % (terminator,)
+ #msg = "Invalid control character {0!r} at".format(terminator)
+ raise JSONDecodeError(msg, s, end)
+ else:
+ _append(terminator)
+ continue
+ try:
+ esc = s[end]
+ except IndexError:
+ raise JSONDecodeError(
+ "Unterminated string starting at", s, begin)
+ # If not a unicode escape sequence, must be in the lookup table
+ if esc != 'u':
+ try:
+ char = _b[esc]
+ except KeyError:
+ msg = "Invalid \\escape: " + repr(esc)
+ raise JSONDecodeError(msg, s, end)
+ end += 1
+ else:
+ # Unicode escape sequence
+ esc = s[end + 1:end + 5]
+ next_end = end + 5
+ if len(esc) != 4:
+ msg = "Invalid \\uXXXX escape"
+ raise JSONDecodeError(msg, s, end)
+ uni = int(esc, 16)
+ # Check for surrogate pair on UCS-4 systems
+ if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
+ msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
+ if not s[end + 5:end + 7] == '\\u':
+ raise JSONDecodeError(msg, s, end)
+ esc2 = s[end + 7:end + 11]
+ if len(esc2) != 4:
+ raise JSONDecodeError(msg, s, end)
+ uni2 = int(esc2, 16)
+ uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
+ next_end += 6
+ char = unichr(uni)
+ end = next_end
+ # Append the unescaped character
+ _append(char)
+ return u''.join(chunks), end
+
+
+# Use speedup if available
+scanstring = c_scanstring or py_scanstring
+
+WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
+WHITESPACE_STR = ' \t\n\r'
+
+def JSONObject((s, end), encoding, strict, scan_once, object_hook,
+ object_pairs_hook, memo=None,
+ _w=WHITESPACE.match, _ws=WHITESPACE_STR):
+ # Backwards compatibility
+ if memo is None:
+ memo = {}
+ memo_get = memo.setdefault
+ pairs = []
+ # Use a slice to prevent IndexError from being raised, the following
+ # check will raise a more specific ValueError if the string is empty
+ nextchar = s[end:end + 1]
+ # Normally we expect nextchar == '"'
+ if nextchar != '"':
+ if nextchar in _ws:
+ end = _w(s, end).end()
+ nextchar = s[end:end + 1]
+ # Trivial empty object
+ if nextchar == '}':
+ if object_pairs_hook is not None:
+ result = object_pairs_hook(pairs)
+ return result, end + 1
+ pairs = {}
+ if object_hook is not None:
+ pairs = object_hook(pairs)
+ return pairs, end + 1
+ elif nextchar != '"':
+ raise JSONDecodeError("Expecting property name", s, end)
+ end += 1
+ while True:
+ key, end = scanstring(s, end, encoding, strict)
+ key = memo_get(key, key)
+
+ # To skip some function call overhead we optimize the fast paths where
+ # the JSON key separator is ": " or just ":".
+ if s[end:end + 1] != ':':
+ end = _w(s, end).end()
+ if s[end:end + 1] != ':':
+ raise JSONDecodeError("Expecting : delimiter", s, end)
+
+ end += 1
+
+ try:
+ if s[end] in _ws:
+ end += 1
+ if s[end] in _ws:
+ end = _w(s, end + 1).end()
+ except IndexError:
+ pass
+
+ try:
+ value, end = scan_once(s, end)
+ except StopIteration:
+ raise JSONDecodeError("Expecting object", s, end)
+ pairs.append((key, value))
+
+ try:
+ nextchar = s[end]
+ if nextchar in _ws:
+ end = _w(s, end + 1).end()
+ nextchar = s[end]
+ except IndexError:
+ nextchar = ''
+ end += 1
+
+ if nextchar == '}':
+ break
+ elif nextchar != ',':
+ raise JSONDecodeError("Expecting , delimiter", s, end - 1)
+
+ try:
+ nextchar = s[end]
+ if nextchar in _ws:
+ end += 1
+ nextchar = s[end]
+ if nextchar in _ws:
+ end = _w(s, end + 1).end()
+ nextchar = s[end]
+ except IndexError:
+ nextchar = ''
+
+ end += 1
+ if nextchar != '"':
+ raise JSONDecodeError("Expecting property name", s, end - 1)
+
+ if object_pairs_hook is not None:
+ result = object_pairs_hook(pairs)
+ return result, end
+ pairs = dict(pairs)
+ if object_hook is not None:
+ pairs = object_hook(pairs)
+ return pairs, end
+
+def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
+ values = []
+ nextchar = s[end:end + 1]
+ if nextchar in _ws:
+ end = _w(s, end + 1).end()
+ nextchar = s[end:end + 1]
+ # Look-ahead for trivial empty array
+ if nextchar == ']':
+ return values, end + 1
+ _append = values.append
+ while True:
+ try:
+ value, end = scan_once(s, end)
+ except StopIteration:
+ raise JSONDecodeError("Expecting object", s, end)
+ _append(value)
+ nextchar = s[end:end + 1]
+ if nextchar in _ws:
+ end = _w(s, end + 1).end()
+ nextchar = s[end:end + 1]
+ end += 1
+ if nextchar == ']':
+ break
+ elif nextchar != ',':
+ raise JSONDecodeError("Expecting , delimiter", s, end)
+
+ try:
+ if s[end] in _ws:
+ end += 1
+ if s[end] in _ws:
+ end = _w(s, end + 1).end()
+ except IndexError:
+ pass
+
+ return values, end
+
+class JSONDecoder(object):
+ """Simple JSON <http://json.org> decoder
+
+ Performs the following translations in decoding by default:
+
+ +---------------+-------------------+
+ | JSON | Python |
+ +===============+===================+
+ | object | dict |
+ +---------------+-------------------+
+ | array | list |
+ +---------------+-------------------+
+ | string | unicode |
+ +---------------+-------------------+
+ | number (int) | int, long |
+ +---------------+-------------------+
+ | number (real) | float |
+ +---------------+-------------------+
+ | true | True |
+ +---------------+-------------------+
+ | false | False |
+ +---------------+-------------------+
+ | null | None |
+ +---------------+-------------------+
+
+ It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
+ their corresponding ``float`` values, which is outside the JSON spec.
+
+ """
+
+ def __init__(self, encoding=None, object_hook=None, parse_float=None,
+ parse_int=None, parse_constant=None, strict=True,
+ object_pairs_hook=None):
+ """
+ *encoding* determines the encoding used to interpret any
+ :class:`str` objects decoded by this instance (``'utf-8'`` by
+ default). It has no effect when decoding :class:`unicode` objects.
+
+ Note that currently only encodings that are a superset of ASCII work,
+ strings of other encodings should be passed in as :class:`unicode`.
+
+ *object_hook*, if specified, will be called with the result of every
+ JSON object decoded and its return value will be used in place of the
+ given :class:`dict`. This can be used to provide custom
+ deserializations (e.g. to support JSON-RPC class hinting).
+
+ *object_pairs_hook* is an optional function that will be called with
+ the result of any object literal decode with an ordered list of pairs.
+ The return value of *object_pairs_hook* will be used instead of the
+ :class:`dict`. This feature can be used to implement custom decoders
+ that rely on the order that the key and value pairs are decoded (for
+ example, :func:`collections.OrderedDict` will remember the order of
+ insertion). If *object_hook* is also defined, the *object_pairs_hook*
+ takes priority.
+
+ *parse_float*, if specified, will be called with the string of every
+ JSON float to be decoded. By default, this is equivalent to
+ ``float(num_str)``. This can be used to use another datatype or parser
+ for JSON floats (e.g. :class:`decimal.Decimal`).
+
+ *parse_int*, if specified, will be called with the string of every
+ JSON int to be decoded. By default, this is equivalent to
+ ``int(num_str)``. This can be used to use another datatype or parser
+ for JSON integers (e.g. :class:`float`).
+
+ *parse_constant*, if specified, will be called with one of the
+ following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This
+ can be used to raise an exception if invalid JSON numbers are
+ encountered.
+
+ *strict* controls the parser's behavior when it encounters an
+ invalid control character in a string. The default setting of
+ ``True`` means that unescaped control characters are parse errors, if
+ ``False`` then control characters will be allowed in strings.
+
+ """
+ self.encoding = encoding
+ self.object_hook = object_hook
+ self.object_pairs_hook = object_pairs_hook
+ self.parse_float = parse_float or float
+ self.parse_int = parse_int or int
+ self.parse_constant = parse_constant or _CONSTANTS.__getitem__
+ self.strict = strict
+ self.parse_object = JSONObject
+ self.parse_array = JSONArray
+ self.parse_string = scanstring
+ self.memo = {}
+ self.scan_once = make_scanner(self)
+
+ def decode(self, s, _w=WHITESPACE.match):
+ """Return the Python representation of ``s`` (a ``str`` or ``unicode``
+ instance containing a JSON document)
+
+ """
+ obj, end = self.raw_decode(s, idx=_w(s, 0).end())
+ end = _w(s, end).end()
+ if end != len(s):
+ raise JSONDecodeError("Extra data", s, end, len(s))
+ return obj
+
+ def raw_decode(self, s, idx=0):
+ """Decode a JSON document from ``s`` (a ``str`` or ``unicode``
+ beginning with a JSON document) and return a 2-tuple of the Python
+ representation and the index in ``s`` where the document ended.
+
+ This can be used to decode a JSON document from a string that may
+ have extraneous data at the end.
+
+ """
+ try:
+ obj, end = self.scan_once(s, idx)
+ except StopIteration:
+ raise JSONDecodeError("No JSON object could be decoded", s, idx)
+ return obj, end
diff --git a/simplejson/encoder.py b/simplejson/encoder.py
new file mode 100644
index 0000000..a7f4eea
--- /dev/null
+++ b/simplejson/encoder.py
@@ -0,0 +1,501 @@
+"""Implementation of JSONEncoder
+"""
+import re
+from decimal import Decimal
+
+def _import_speedups():
+ try:
+ from simplejson import _speedups
+ return _speedups.encode_basestring_ascii, _speedups.make_encoder
+ except ImportError:
+ return None, None
+c_encode_basestring_ascii, c_make_encoder = _import_speedups()
+
+from simplejson.decoder import PosInf
+
+ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
+ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
+HAS_UTF8 = re.compile(r'[\x80-\xff]')
+ESCAPE_DCT = {
+ '\\': '\\\\',
+ '"': '\\"',
+ '\b': '\\b',
+ '\f': '\\f',
+ '\n': '\\n',
+ '\r': '\\r',
+ '\t': '\\t',
+}
+for i in range(0x20):
+ #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
+ ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+
+FLOAT_REPR = repr
+
+def encode_basestring(s):
+ """Return a JSON representation of a Python string
+
+ """
+ if isinstance(s, str) and HAS_UTF8.search(s) is not None:
+ s = s.decode('utf-8')
+ def replace(match):
+ return ESCAPE_DCT[match.group(0)]
+ return u'"' + ESCAPE.sub(replace, s) + u'"'
+
+
+def py_encode_basestring_ascii(s):
+ """Return an ASCII-only JSON representation of a Python string
+
+ """
+ if isinstance(s, str) and HAS_UTF8.search(s) is not None:
+ s = s.decode('utf-8')
+ def replace(match):
+ s = match.group(0)
+ try:
+ return ESCAPE_DCT[s]
+ except KeyError:
+ n = ord(s)
+ if n < 0x10000:
+ #return '\\u{0:04x}'.format(n)
+ return '\\u%04x' % (n,)
+ else:
+ # surrogate pair
+ n -= 0x10000
+ s1 = 0xd800 | ((n >> 10) & 0x3ff)
+ s2 = 0xdc00 | (n & 0x3ff)
+ #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
+ return '\\u%04x\\u%04x' % (s1, s2)
+ return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
+
+
+encode_basestring_ascii = (
+ c_encode_basestring_ascii or py_encode_basestring_ascii)
+
+class JSONEncoder(object):
+ """Extensible JSON <http://json.org> encoder for Python data structures.
+
+ Supports the following objects and types by default:
+
+ +-------------------+---------------+
+ | Python | JSON |
+ +===================+===============+
+ | dict | object |
+ +-------------------+---------------+
+ | list, tuple | array |
+ +-------------------+---------------+
+ | str, unicode | string |
+ +-------------------+---------------+
+ | int, long, float | number |
+ +-------------------+---------------+
+ | True | true |
+ +-------------------+---------------+
+ | False | false |
+ +-------------------+---------------+
+ | None | null |
+ +-------------------+---------------+
+
+ To extend this to recognize other objects, subclass and implement a
+ ``.default()`` method with another method that returns a serializable
+ object for ``o`` if possible, otherwise it should call the superclass
+ implementation (to raise ``TypeError``).
+
+ """
+ item_separator = ', '
+ key_separator = ': '
+ def __init__(self, skipkeys=False, ensure_ascii=True,
+ check_circular=True, allow_nan=True, sort_keys=False,
+ indent=None, separators=None, encoding='utf-8', default=None,
+ use_decimal=False):
+ """Constructor for JSONEncoder, with sensible defaults.
+
+ If skipkeys is false, then it is a TypeError to attempt
+ encoding of keys that are not str, int, long, float or None. If
+ skipkeys is True, such items are simply skipped.
+
+ If ensure_ascii is true, the output is guaranteed to be str
+ objects with all incoming unicode characters escaped. If
+ ensure_ascii is false, the output will be unicode object.
+
+ If check_circular is true, then lists, dicts, and custom encoded
+ objects will be checked for circular references during encoding to
+ prevent an infinite recursion (which would cause an OverflowError).
+ Otherwise, no such check takes place.
+
+ If allow_nan is true, then NaN, Infinity, and -Infinity will be
+ encoded as such. This behavior is not JSON specification compliant,
+ but is consistent with most JavaScript based encoders and decoders.
+ Otherwise, it will be a ValueError to encode such floats.
+
+ If sort_keys is true, then the output of dictionaries will be
+ sorted by key; this is useful for regression tests to ensure
+ that JSON serializations can be compared on a day-to-day basis.
+
+ If indent is a string, then JSON array elements and object members
+ will be pretty-printed with a newline followed by that string repeated
+ for each level of nesting. ``None`` (the default) selects the most compact
+ representation without any newlines. For backwards compatibility with
+ versions of simplejson earlier than 2.1.0, an integer is also accepted
+ and is converted to a string with that many spaces.
+
+ If specified, separators should be a (item_separator, key_separator)
+ tuple. The default is (', ', ': '). To get the most compact JSON
+ representation you should specify (',', ':') to eliminate whitespace.
+
+ If specified, default is a function that gets called for objects
+ that can't otherwise be serialized. It should return a JSON encodable
+ version of the object or raise a ``TypeError``.
+
+ If encoding is not None, then all input strings will be
+ transformed into unicode using that encoding prior to JSON-encoding.
+ The default is UTF-8.
+
+ If use_decimal is true (not the default), ``decimal.Decimal`` will
+ be supported directly by the encoder. For the inverse, decode JSON
+ with ``parse_float=decimal.Decimal``.
+
+ """
+
+ self.skipkeys = skipkeys
+ self.ensure_ascii = ensure_ascii
+ self.check_circular = check_circular
+ self.allow_nan = allow_nan
+ self.sort_keys = sort_keys
+ self.use_decimal = use_decimal
+ if isinstance(indent, (int, long)):
+ indent = ' ' * indent
+ self.indent = indent
+ if separators is not None:
+ self.item_separator, self.key_separator = separators
+ if default is not None:
+ self.default = default
+ self.encoding = encoding
+
+ def default(self, o):
+ """Implement this method in a subclass such that it returns
+ a serializable object for ``o``, or calls the base implementation
+ (to raise a ``TypeError``).
+
+ For example, to support arbitrary iterators, you could
+ implement default like this::
+
+ def default(self, o):
+ try:
+ iterable = iter(o)
+ except TypeError:
+ pass
+ else:
+ return list(iterable)
+ return JSONEncoder.default(self, o)
+
+ """
+ raise TypeError(repr(o) + " is not JSON serializable")
+
+ def encode(self, o):
+ """Return a JSON string representation of a Python data structure.
+
+ >>> from simplejson import JSONEncoder
+ >>> JSONEncoder().encode({"foo": ["bar", "baz"]})
+ '{"foo": ["bar", "baz"]}'
+
+ """
+ # This is for extremely simple cases and benchmarks.
+ if isinstance(o, basestring):
+ if isinstance(o, str):
+ _encoding = self.encoding
+ if (_encoding is not None
+ and not (_encoding == 'utf-8')):
+ o = o.decode(_encoding)
+ if self.ensure_ascii:
+ return encode_basestring_ascii(o)
+ else:
+ return encode_basestring(o)
+ # This doesn't pass the iterator directly to ''.join() because the
+ # exceptions aren't as detailed. The list call should be roughly
+ # equivalent to the PySequence_Fast that ''.join() would do.
+ chunks = self.iterencode(o, _one_shot=True)
+ if not isinstance(chunks, (list, tuple)):
+ chunks = list(chunks)
+ if self.ensure_ascii:
+ return ''.join(chunks)
+ else:
+ return u''.join(chunks)
+
+ def iterencode(self, o, _one_shot=False):
+ """Encode the given object and yield each string
+ representation as available.
+
+ For example::
+
+ for chunk in JSONEncoder().iterencode(bigobject):
+ mysocket.write(chunk)
+
+ """
+ if self.check_circular:
+ markers = {}
+ else:
+ markers = None
+ if self.ensure_ascii:
+ _encoder = encode_basestring_ascii
+ else:
+ _encoder = encode_basestring
+ if self.encoding != 'utf-8':
+ def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
+ if isinstance(o, str):
+ o = o.decode(_encoding)
+ return _orig_encoder(o)
+
+ def floatstr(o, allow_nan=self.allow_nan,
+ _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf):
+ # Check for specials. Note that this type of test is processor
+ # and/or platform-specific, so do tests which don't depend on
+ # the internals.
+
+ if o != o:
+ text = 'NaN'
+ elif o == _inf:
+ text = 'Infinity'
+ elif o == _neginf:
+ text = '-Infinity'
+ else:
+ return _repr(o)
+
+ if not allow_nan:
+ raise ValueError(
+ "Out of range float values are not JSON compliant: " +
+ repr(o))
+
+ return text
+
+
+ key_memo = {}
+ if (_one_shot and c_make_encoder is not None
+ and self.indent is None and not self.sort_keys):
+ _iterencode = c_make_encoder(
+ markers, self.default, _encoder, self.indent,
+ self.key_separator, self.item_separator, self.sort_keys,
+ self.skipkeys, self.allow_nan, key_memo, self.use_decimal)
+ else:
+ _iterencode = _make_iterencode(
+ markers, self.default, _encoder, self.indent, floatstr,
+ self.key_separator, self.item_separator, self.sort_keys,
+ self.skipkeys, _one_shot, self.use_decimal)
+ try:
+ return _iterencode(o, 0)
+ finally:
+ key_memo.clear()
+
+
+class JSONEncoderForHTML(JSONEncoder):
+ """An encoder that produces JSON safe to embed in HTML.
+
+ To embed JSON content in, say, a script tag on a web page, the
+ characters &, < and > should be escaped. They cannot be escaped
+ with the usual entities (e.g. &) because they are not expanded
+ within <script> tags.
+ """
+
+ def encode(self, o):
+ # Override JSONEncoder.encode because it has hacks for
+ # performance that make things more complicated.
+ chunks = self.iterencode(o, True)
+ if self.ensure_ascii:
+ return ''.join(chunks)
+ else:
+ return u''.join(chunks)
+
+ def iterencode(self, o, _one_shot=False):
+ chunks = super(JSONEncoderForHTML, self).iterencode(o, _one_shot)
+ for chunk in chunks:
+ chunk = chunk.replace('&', '\\u0026')
+ chunk = chunk.replace('<', '\\u003c')
+ chunk = chunk.replace('>', '\\u003e')
+ yield chunk
+
+
+def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
+ _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
+ _use_decimal,
+ ## HACK: hand-optimized bytecode; turn globals into locals
+ False=False,
+ True=True,
+ ValueError=ValueError,
+ basestring=basestring,
+ Decimal=Decimal,
+ dict=dict,
+ float=float,
+ id=id,
+ int=int,
+ isinstance=isinstance,
+ list=list,
+ long=long,
+ str=str,
+ tuple=tuple,
+ ):
+
+ def _iterencode_list(lst, _current_indent_level):
+ if not lst:
+ yield '[]'
+ return
+ if markers is not None:
+ markerid = id(lst)
+ if markerid in markers:
+ raise ValueError("Circular reference detected")
+ markers[markerid] = lst
+ buf = '['
+ if _indent is not None:
+ _current_indent_level += 1
+ newline_indent = '\n' + (_indent * _current_indent_level)
+ separator = _item_separator + newline_indent
+ buf += newline_indent
+ else:
+ newline_indent = None
+ separator = _item_separator
+ first = True
+ for value in lst:
+ if first:
+ first = False
+ else:
+ buf = separator
+ if isinstance(value, basestring):
+ yield buf + _encoder(value)
+ elif value is None:
+ yield buf + 'null'
+ elif value is True:
+ yield buf + 'true'
+ elif value is False:
+ yield buf + 'false'
+ elif isinstance(value, (int, long)):
+ yield buf + str(value)
+ elif isinstance(value, float):
+ yield buf + _floatstr(value)
+ elif _use_decimal and isinstance(value, Decimal):
+ yield buf + str(value)
+ else:
+ yield buf
+ if isinstance(value, (list, tuple)):
+ chunks = _iterencode_list(value, _current_indent_level)
+ elif isinstance(value, dict):
+ chunks = _iterencode_dict(value, _current_indent_level)
+ else:
+ chunks = _iterencode(value, _current_indent_level)
+ for chunk in chunks:
+ yield chunk
+ if newline_indent is not None:
+ _current_indent_level -= 1
+ yield '\n' + (_indent * _current_indent_level)
+ yield ']'
+ if markers is not None:
+ del markers[markerid]
+
+ def _iterencode_dict(dct, _current_indent_level):
+ if not dct:
+ yield '{}'
+ return
+ if markers is not None:
+ markerid = id(dct)
+ if markerid in markers:
+ raise ValueError("Circular reference detected")
+ markers[markerid] = dct
+ yield '{'
+ if _indent is not None:
+ _current_indent_level += 1
+ newline_indent = '\n' + (_indent * _current_indent_level)
+ item_separator = _item_separator + newline_indent
+ yield newline_indent
+ else:
+ newline_indent = None
+ item_separator = _item_separator
+ first = True
+ if _sort_keys:
+ items = dct.items()
+ items.sort(key=lambda kv: kv[0])
+ else:
+ items = dct.iteritems()
+ for key, value in items:
+ if isinstance(key, basestring):
+ pass
+ # JavaScript is weakly typed for these, so it makes sense to
+ # also allow them. Many encoders seem to do something like this.
+ elif isinstance(key, float):
+ key = _floatstr(key)
+ elif key is True:
+ key = 'true'
+ elif key is False:
+ key = 'false'
+ elif key is None:
+ key = 'null'
+ elif isinstance(key, (int, long)):
+ key = str(key)
+ elif _skipkeys:
+ continue
+ else:
+ raise TypeError("key " + repr(key) + " is not a string")
+ if first:
+ first = False
+ else:
+ yield item_separator
+ yield _encoder(key)
+ yield _key_separator
+ if isinstance(value, basestring):
+ yield _encoder(value)
+ elif value is None:
+ yield 'null'
+ elif value is True:
+ yield 'true'
+ elif value is False:
+ yield 'false'
+ elif isinstance(value, (int, long)):
+ yield str(value)
+ elif isinstance(value, float):
+ yield _floatstr(value)
+ elif _use_decimal and isinstance(value, Decimal):
+ yield str(value)
+ else:
+ if isinstance(value, (list, tuple)):
+ chunks = _iterencode_list(value, _current_indent_level)
+ elif isinstance(value, dict):
+ chunks = _iterencode_dict(value, _current_indent_level)
+ else:
+ chunks = _iterencode(value, _current_indent_level)
+ for chunk in chunks:
+ yield chunk
+ if newline_indent is not None:
+ _current_indent_level -= 1
+ yield '\n' + (_indent * _current_indent_level)
+ yield '}'
+ if markers is not None:
+ del markers[markerid]
+
+ def _iterencode(o, _current_indent_level):
+ if isinstance(o, basestring):
+ yield _encoder(o)
+ elif o is None:
+ yield 'null'
+ elif o is True:
+ yield 'true'
+ elif o is False:
+ yield 'false'
+ elif isinstance(o, (int, long)):
+ yield str(o)
+ elif isinstance(o, float):
+ yield _floatstr(o)
+ elif isinstance(o, (list, tuple)):
+ for chunk in _iterencode_list(o, _current_indent_level):
+ yield chunk
+ elif isinstance(o, dict):
+ for chunk in _iterencode_dict(o, _current_indent_level):
+ yield chunk
+ elif _use_decimal and isinstance(o, Decimal):
+ yield str(o)
+ else:
+ if markers is not None:
+ markerid = id(o)
+ if markerid in markers:
+ raise ValueError("Circular reference detected")
+ markers[markerid] = o
+ o = _default(o)
+ for chunk in _iterencode(o, _current_indent_level):
+ yield chunk
+ if markers is not None:
+ del markers[markerid]
+
+ return _iterencode
diff --git a/simplejson/ordered_dict.py b/simplejson/ordered_dict.py
new file mode 100644
index 0000000..87ad888
--- /dev/null
+++ b/simplejson/ordered_dict.py
@@ -0,0 +1,119 @@
+"""Drop-in replacement for collections.OrderedDict by Raymond Hettinger
+
+http://code.activestate.com/recipes/576693/
+
+"""
+from UserDict import DictMixin
+
+# Modified from original to support Python 2.4, see
+# http://code.google.com/p/simplejson/issues/detail?id=53
+try:
+ all
+except NameError:
+ def all(seq):
+ for elem in seq:
+ if not elem:
+ return False
+ return True
+
+class OrderedDict(dict, DictMixin):
+
+ def __init__(self, *args, **kwds):
+ if len(args) > 1:
+ raise TypeError('expected at most 1 arguments, got %d' % len(args))
+ try:
+ self.__end
+ except AttributeError:
+ self.clear()
+ self.update(*args, **kwds)
+
+ def clear(self):
+ self.__end = end = []
+ end += [None, end, end] # sentinel node for doubly linked list
+ self.__map = {} # key --> [key, prev, next]
+ dict.clear(self)
+
+ def __setitem__(self, key, value):
+ if key not in self:
+ end = self.__end
+ curr = end[1]
+ curr[2] = end[1] = self.__map[key] = [key, curr, end]
+ dict.__setitem__(self, key, value)
+
+ def __delitem__(self, key):
+ dict.__delitem__(self, key)
+ key, prev, next = self.__map.pop(key)
+ prev[2] = next
+ next[1] = prev
+
+ def __iter__(self):
+ end = self.__end
+ curr = end[2]
+ while curr is not end:
+ yield curr[0]
+ curr = curr[2]
+
+ def __reversed__(self):
+ end = self.__end
+ curr = end[1]
+ while curr is not end:
+ yield curr[0]
+ curr = curr[1]
+
+ def popitem(self, last=True):
+ if not self:
+ raise KeyError('dictionary is empty')
+ # Modified from original to support Python 2.4, see
+ # http://code.google.com/p/simplejson/issues/detail?id=53
+ if last:
+ key = reversed(self).next()
+ else:
+ key = iter(self).next()
+ value = self.pop(key)
+ return key, value
+
+ def __reduce__(self):
+ items = [[k, self[k]] for k in self]
+ tmp = self.__map, self.__end
+ del self.__map, self.__end
+ inst_dict = vars(self).copy()
+ self.__map, self.__end = tmp
+ if inst_dict:
+ return (self.__class__, (items,), inst_dict)
+ return self.__class__, (items,)
+
+ def keys(self):
+ return list(self)
+
+ setdefault = DictMixin.setdefault
+ update = DictMixin.update
+ pop = DictMixin.pop
+ values = DictMixin.values
+ items = DictMixin.items
+ iterkeys = DictMixin.iterkeys
+ itervalues = DictMixin.itervalues
+ iteritems = DictMixin.iteritems
+
+ def __repr__(self):
+ if not self:
+ return '%s()' % (self.__class__.__name__,)
+ return '%s(%r)' % (self.__class__.__name__, self.items())
+
+ def copy(self):
+ return self.__class__(self)
+
+ @classmethod
+ def fromkeys(cls, iterable, value=None):
+ d = cls()
+ for key in iterable:
+ d[key] = value
+ return d
+
+ def __eq__(self, other):
+ if isinstance(other, OrderedDict):
+ return len(self)==len(other) and \
+ all(p==q for p, q in zip(self.items(), other.items()))
+ return dict.__eq__(self, other)
+
+ def __ne__(self, other):
+ return not self == other
diff --git a/simplejson/scanner.py b/simplejson/scanner.py
new file mode 100644
index 0000000..54593a3
--- /dev/null
+++ b/simplejson/scanner.py
@@ -0,0 +1,77 @@
+"""JSON token scanner
+"""
+import re
+def _import_c_make_scanner():
+ try:
+ from simplejson._speedups import make_scanner
+ return make_scanner
+ except ImportError:
+ return None
+c_make_scanner = _import_c_make_scanner()
+
+__all__ = ['make_scanner']
+
+NUMBER_RE = re.compile(
+ r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
+ (re.VERBOSE | re.MULTILINE | re.DOTALL))
+
+def py_make_scanner(context):
+ parse_object = context.parse_object
+ parse_array = context.parse_array
+ parse_string = context.parse_string
+ match_number = NUMBER_RE.match
+ encoding = context.encoding
+ strict = context.strict
+ parse_float = context.parse_float
+ parse_int = context.parse_int
+ parse_constant = context.parse_constant
+ object_hook = context.object_hook
+ object_pairs_hook = context.object_pairs_hook
+ memo = context.memo
+
+ def _scan_once(string, idx):
+ try:
+ nextchar = string[idx]
+ except IndexError:
+ raise StopIteration
+
+ if nextchar == '"':
+ return parse_string(string, idx + 1, encoding, strict)
+ elif nextchar == '{':
+ return parse_object((string, idx + 1), encoding, strict,
+ _scan_once, object_hook, object_pairs_hook, memo)
+ elif nextchar == '[':
+ return parse_array((string, idx + 1), _scan_once)
+ elif nextchar == 'n' and string[idx:idx + 4] == 'null':
+ return None, idx + 4
+ elif nextchar == 't' and string[idx:idx + 4] == 'true':
+ return True, idx + 4
+ elif nextchar == 'f' and string[idx:idx + 5] == 'false':
+ return False, idx + 5
+
+ m = match_number(string, idx)
+ if m is not None:
+ integer, frac, exp = m.groups()
+ if frac or exp:
+ res = parse_float(integer + (frac or '') + (exp or ''))
+ else:
+ res = parse_int(integer)
+ return res, m.end()
+ elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
+ return parse_constant('NaN'), idx + 3
+ elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
+ return parse_constant('Infinity'), idx + 8
+ elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
+ return parse_constant('-Infinity'), idx + 9
+ else:
+ raise StopIteration
+
+ def scan_once(string, idx):
+ try:
+ return _scan_once(string, idx)
+ finally:
+ memo.clear()
+
+ return scan_once
+
+make_scanner = c_make_scanner or py_make_scanner
diff --git a/simplejson/tool.py b/simplejson/tool.py
new file mode 100644
index 0000000..73370db
--- /dev/null
+++ b/simplejson/tool.py
@@ -0,0 +1,39 @@
+r"""Command-line tool to validate and pretty-print JSON
+
+Usage::
+
+ $ echo '{"json":"obj"}' | python -m simplejson.tool
+ {
+ "json": "obj"
+ }
+ $ echo '{ 1.2:3.4}' | python -m simplejson.tool
+ Expecting property name: line 1 column 2 (char 2)
+
+"""
+import sys
+import simplejson as json
+
+def main():
+ if len(sys.argv) == 1:
+ infile = sys.stdin
+ outfile = sys.stdout
+ elif len(sys.argv) == 2:
+ infile = open(sys.argv[1], 'rb')
+ outfile = sys.stdout
+ elif len(sys.argv) == 3:
+ infile = open(sys.argv[1], 'rb')
+ outfile = open(sys.argv[2], 'wb')
+ else:
+ raise SystemExit(sys.argv[0] + " [infile [outfile]]")
+ try:
+ obj = json.load(infile,
+ object_pairs_hook=json.OrderedDict,
+ use_decimal=True)
+ except ValueError, e:
+ raise SystemExit(e)
+ json.dump(obj, outfile, sort_keys=True, indent=' ', use_decimal=True)
+ outfile.write('\n')
+
+
+if __name__ == '__main__':
+ main()
