gh-108885: Report the examples of a doctest run by unittest (GH-155377)
DocTestCase ran its examples with verbose=False, so there was no way to ask
unittest for the details which doctest reports on its own. It now takes the
verbosity from the test result, and reports every example if the test runner
is asked for more than the test names. They are written to the stream of the
test runner, so that they are not lost when it buffers the output of the test.
To make this reachable:
* unittest.TestResult has now a verbosity attribute, which it accepted but
ignored. The test runner sets it, because a result class is free to filter
what its constructor gets.
* The unittest -v option is now counted, so that -vv means 3.
* The verbosity of regrtest is one less, so it is translated where the test
runner is created: -v reports the test names, as before, and -vv reports
also the examples.
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst
index 3298697..4b61c07 100644
--- a/Doc/library/doctest.rst
+++ b/Doc/library/doctest.rst
@@ -1162,6 +1162,12 @@
.. versionchanged:: 3.15
Run each example as a :ref:`subtest <subtests>`.
+ .. versionchanged:: next
+ Report every example, as in verbose mode, if the test runner reports
+ more than the test names, i.e. its
+ :attr:`~unittest.TestResult.verbosity` is 3 or higher (for example
+ with ``python -m unittest -vv``).
+
Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out
of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a
subclass of :class:`unittest.TestCase`. :class:`!DocTestCase` isn't documented
diff --git a/Doc/library/test.rst b/Doc/library/test.rst
index 72ffc2a..765811e 100644
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -232,9 +232,10 @@
.. data:: verbose
- ``True`` when verbose output is enabled. Should be checked when more
- detailed information is desired about a running test. *verbose* is set by
- :mod:`test.regrtest`.
+ How verbose the output is: the number of :option:`!-v` options which
+ :mod:`test.regrtest` was run with, and therefore ``0`` when verbose output
+ is not enabled. Should be checked when more detailed information is
+ desired about a running test.
.. data:: is_jython
diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst
index 7afcdb3..e1bc32f 100644
--- a/Doc/library/unittest.rst
+++ b/Doc/library/unittest.rst
@@ -180,6 +180,9 @@
python -m unittest -v test_module
+Repeat it for even more detail: ``-vv`` reports also the individual examples
+of a :mod:`doctest`.
+
When executed without arguments :ref:`unittest-test-discovery` is started::
python -m unittest
@@ -291,7 +294,11 @@
.. option:: -v, --verbose
- Verbose output
+ Verbose output. May be repeated: ``-vv`` reports also the individual
+ examples of a :mod:`doctest`.
+
+ .. versionchanged:: next
+ The option can be repeated.
.. option:: -s, --start-directory directory
@@ -2149,6 +2156,15 @@
.. versionadded:: 3.5
+ .. attribute:: verbosity
+
+ The level of details which the test runner reports: ``0`` -- quiet,
+ ``1`` -- progress dots, ``2`` -- test names, ``3`` -- also the
+ individual examples of a :mod:`doctest`. A test runner is expected to
+ set it to its own verbosity.
+
+ .. versionadded:: next
+
.. method:: wasSuccessful()
Return ``True`` if all tests run so far have passed, otherwise returns
diff --git a/Lib/doctest.py b/Lib/doctest.py
index d5541ab..a8f0886 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -1201,6 +1201,18 @@ def _find_lineno(self, obj, source_lines):
## 5. DocTest Runner
######################################################################
+def _make_output_function(stream):
+ """Return a function writing to *stream*, whatever it can encode."""
+ encoding = getattr(stream, 'encoding', None)
+ if encoding is None or encoding.lower() == 'utf-8':
+ return stream.write
+ def out(s):
+ # Use backslashreplace error handling on write
+ s = str(s.encode(encoding, 'backslashreplace'), encoding)
+ stream.write(s)
+ return out
+
+
class DocTestRunner:
"""
A class used to run DocTest test cases, and accumulate statistics.
@@ -1561,14 +1573,7 @@ def run(self, test, compileflags=None, out=None, clear_globs=True):
save_stdout = sys.stdout
if out is None:
- encoding = save_stdout.encoding
- if encoding is None or encoding.lower() == 'utf-8':
- out = save_stdout.write
- else:
- # Use backslashreplace error handling on write
- def out(s):
- s = str(s.encode(encoding, 'backslashreplace'), encoding)
- save_stdout.write(s)
+ out = _make_output_function(save_stdout)
sys.stdout = self._fakeout
# Patch pdb.set_trace to restore sys.stdout during interactive
@@ -2322,6 +2327,9 @@ def report_skip(self, out, test, example):
unittest.case._addSkip(self._test_result, self._subTest(), '')
def report_success(self, out, test, example, got):
+ # Report "ok" if verbose, to close what report_start() opened. A
+ # failed or skipped example is reported by the test result instead.
+ super().report_success(out, test, example, got)
self._test_result.addSubTest(self._test_case, self._subTest(), None)
def report_unexpected_exception(self, out, test, example, exc_info):
@@ -2401,10 +2409,23 @@ def runTest(self):
if getattr(result, 'failfast', False):
optionflags |= FAIL_FAST
+ # Report every example only if the test runner is asked for more than
+ # the test names it reports at verbosity 2. Write them to its stream,
+ # so that they are not swallowed by result.buffer.
+ verbose = getattr(result, 'verbosity', 1) >= 3
+ stream = getattr(result, 'stream', None)
+ out = None
+ if verbose and stream is not None:
+ out = _make_output_function(stream)
+ if test.examples and not getattr(result, '_newline', True):
+ # End the line which startTest() left open.
+ out('\n')
+ result._newline = True
+
runner = _DocTestCaseRunner(optionflags=optionflags,
- checker=self._dt_checker, verbose=False,
+ checker=self._dt_checker, verbose=verbose,
test_case=self, test_result=result)
- results = runner.run(test, clear_globs=False)
+ results = runner.run(test, out=out, clear_globs=False)
if results.skipped == results.attempted:
raise unittest.SkipTest("all examples were skipped")
diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py
index 1820f35..605f1f4 100644
--- a/Lib/test/libregrtest/testresult.py
+++ b/Lib/test/libregrtest/testresult.py
@@ -16,7 +16,7 @@ class RegressionTestResult(unittest.TextTestResult):
def __init__(self, stream, descriptions, verbosity):
super().__init__(stream=stream, descriptions=descriptions,
- verbosity=2 if verbosity else 0)
+ verbosity=verbosity)
self.buffer = True
if self.USE_XML:
from xml.etree import ElementTree as ET
@@ -150,10 +150,12 @@ def run(self, test):
def get_test_runner_class(verbosity, buffer=False):
if verbosity:
+ # The verbosity of regrtest is one less than the verbosity of
+ # unittest: -v reports the test names, -vv also the doctest examples.
return functools.partial(unittest.TextTestRunner,
resultclass=RegressionTestResult,
buffer=buffer,
- verbosity=verbosity)
+ verbosity=verbosity + 1)
return functools.partial(QuietRegressionTestRunner, buffer=buffer)
def get_test_runner(stream, verbosity, capture_output=False):
diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py
index b125693..776ad83 100644
--- a/Lib/test/test_doctest/test_doctest.py
+++ b/Lib/test/test_doctest/test_doctest.py
@@ -6,6 +6,7 @@
from test.support import import_helper
import doctest
import functools
+import io
import os
import sys
import importlib
@@ -469,7 +470,7 @@ def basics(): r"""
>>> tests = finder.find(sample_func)
>>> print(tests) # doctest: +ELLIPSIS
- [<DocTest sample_func from test_doctest.py:36 (1 example)>]
+ [<DocTest sample_func from test_doctest.py:37 (1 example)>]
The exact name depends on how test_doctest was invoked, so allow for
leading path components.
@@ -803,6 +804,59 @@ def myfunc():
self.assertEqual((x, y), (2, 3))
+class TestDocTestSuiteVerbosity(unittest.TestCase):
+
+ def run_suite(self, module='test.test_doctest.sample_doctest', **kwargs):
+ """Return what the test runner wrote and what leaked to stdout."""
+ suite = doctest.DocTestSuite(module)
+ stream = io.StringIO()
+ stdout = io.StringIO()
+ with contextlib.redirect_stdout(stdout):
+ unittest.TextTestRunner(stream=stream, **kwargs).run(suite)
+ return stream.getvalue(), stdout.getvalue()
+
+ def test_quiet(self):
+ for verbosity in range(3):
+ with self.subTest(verbosity=verbosity):
+ output, stdout = self.run_suite(verbosity=verbosity)
+ self.assertNotIn('Trying:', output)
+ self.assertNotIn('Expecting:', output)
+ self.assertEqual(stdout, '')
+
+ def test_verbose(self):
+ output, stdout = self.run_suite(verbosity=3)
+ self.assertIn('Trying:\n 2+2\n', output)
+ self.assertIn('Expecting:\n 4\n', output)
+ self.assertIn('\nok\n', output)
+ # Reported to the stream of the test runner, not to the stdout.
+ self.assertEqual(stdout, '')
+
+ def test_verbose_buffered(self):
+ # result.buffer replaces sys.stdout, which would swallow the examples.
+ output, stdout = self.run_suite(verbosity=3, buffer=True)
+ self.assertIn('Trying:\n 2+2\n', output)
+ self.assertEqual(stdout, '')
+
+ def test_verbose_failure_not_duplicated(self):
+ module = 'test.test_doctest.sample_doctest_errors'
+ quiet, _ = self.run_suite(module, verbosity=2)
+ verbose, _ = self.run_suite(module, verbosity=3)
+ self.assertIn('Trying:', verbose)
+ self.assertNotIn('Trying:', quiet)
+ # Reporting the examples does not report the failures once more.
+ self.assertEqual(verbose.count('Failed example:'),
+ quiet.count('Failed example:'))
+ self.assertGreater(quiet.count('Failed example:'), 0)
+
+ def test_plain_result(self):
+ # A result which is not from a text test runner has no stream.
+ suite = doctest.DocTestSuite('test.test_doctest.sample_doctest')
+ stdout = io.StringIO()
+ with contextlib.redirect_stdout(stdout):
+ suite.run(unittest.TestResult())
+ self.assertEqual(stdout.getvalue(), '')
+
+
class TestDocTestFinder(unittest.TestCase):
def test_issue35753(self):
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index f4baa96..c966f86 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -2184,6 +2184,18 @@ def load_tests(loader, tests, pattern):
failed=[testname],
parallel=True,
stats=TestStats(1, 2, 1))
+ # A single -v reports the test names, not the examples.
+ self.assertNotIn('Trying:', output)
+
+ # -vv reports every example, without changing what is run.
+ output = self.run_tests("--fail-env-changed", "-vv", "-j1", testname,
+ exitcode=EXITCODE_BAD_TEST)
+ self.check_executed_tests(output, [testname],
+ failed=[testname],
+ parallel=True,
+ stats=TestStats(1, 2, 1))
+ self.assertIn('Trying:\n 1 + 1\n', output)
+ self.assertIn('Expecting:\n 2\n', output)
def _check_random_seed(self, run_workers: bool):
# gh-109276: When -r/--randomize is used, random.seed() is called
diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py
index 38c9779..da184bd 100644
--- a/Lib/test/test_unittest/test_discovery.py
+++ b/Lib/test/test_unittest/test_discovery.py
@@ -635,7 +635,7 @@ def test_command_line_handling_discover_by_default_with_options(self):
program._do_discovery = args.append
program.parseArgs(['something', '-v', '-b', '-v', '-c', '-f'])
self.assertEqual(args, [[]])
- self.assertEqual(program.verbosity, 2)
+ self.assertEqual(program.verbosity, 3) # -v is passed twice
self.assertIs(program.buffer, True)
self.assertIs(program.catchbreak, True)
self.assertIs(program.failfast, True)
diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py
index 8ed9237..a9a73f0 100644
--- a/Lib/test/test_unittest/test_program.py
+++ b/Lib/test/test_unittest/test_program.py
@@ -267,6 +267,32 @@ def testVerbosity(self):
program.parseArgs([None, opt])
self.assertEqual(program.verbosity, 2)
+ # -v can be repeated to ask for more details.
+ for args, verbosity in (
+ (['-vv'], 3),
+ (['-v', '-v'], 3),
+ (['--verbose', '--verbose'], 3),
+ (['-vvv'], 4),
+ # -q overrides any number of -v.
+ (['-v', '-q'], 0),
+ ):
+ with self.subTest(args=args):
+ program.verbosity = 1
+ program.parseArgs([None, *args])
+ self.assertEqual(program.verbosity, verbosity)
+
+ def testVerbosityCountedOnce(self):
+ # "python -m unittest -v" falls back to test discovery, which parses
+ # arguments again: -v must not be counted twice.
+ program = self.program
+ program.verbosity = 1
+ program.parseArgs([None, '-v'])
+ self.assertEqual(program.verbosity, 2)
+
+ program.verbosity = 1
+ program.parseArgs([None, 'discover', '-vv'])
+ self.assertEqual(program.verbosity, 3)
+
def testBufferCatchFailfast(self):
program = self.program
for arg, attr in (('buffer', 'buffer'), ('failfast', 'failfast'),
diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py
index 3f44e61..cb591c2 100644
--- a/Lib/test/test_unittest/test_result.py
+++ b/Lib/test/test_unittest/test_result.py
@@ -54,6 +54,13 @@ def test_init(self):
self.assertEqual(result.shouldStop, False)
self.assertIsNone(result._stdout_buffer)
self.assertIsNone(result._stderr_buffer)
+ self.assertEqual(result.verbosity, 1)
+
+ def test_init_verbosity(self):
+ for verbosity in range(4):
+ with self.subTest(verbosity=verbosity):
+ result = unittest.TestResult(None, None, verbosity)
+ self.assertEqual(result.verbosity, verbosity)
# "This method can be called to signal that the set of tests being
# run should be aborted by setting the TestResult's shouldStop
diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py
index a47e2eb..195d5e9 100644
--- a/Lib/test/test_unittest/test_runner.py
+++ b/Lib/test/test_unittest/test_runner.py
@@ -1363,6 +1363,35 @@ def MockResultClass(*args):
expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY)
self.assertEqual(runner._makeResult(), expectedresult)
+ def test_verbosity_set_on_result(self):
+ class Suite:
+ def __call__(self, result):
+ pass
+
+ for verbosity in range(4):
+ with self.subTest(verbosity=verbosity):
+ runner = unittest.TextTestRunner(io.StringIO(),
+ verbosity=verbosity)
+ result = runner.run(Suite())
+ self.assertEqual(result.verbosity, verbosity)
+
+ def test_verbosity_set_on_filtering_result(self):
+ # A result class is free to filter the verbosity which its
+ # constructor gets, as test.libregrtest does.
+ class FilteringResult(unittest.TextTestResult):
+ def __init__(self, stream, descriptions, verbosity):
+ super().__init__(stream, descriptions,
+ 2 if verbosity else 0)
+
+ class Suite:
+ def __call__(self, result):
+ pass
+
+ runner = unittest.TextTestRunner(io.StringIO(), verbosity=3,
+ resultclass=FilteringResult)
+ result = runner.run(Suite())
+ self.assertEqual(result.verbosity, 3)
+
@support.force_not_colorized
@support.requires_subprocess()
def test_warnings(self):
diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py
index 6eeebf9..850c825 100644
--- a/Lib/unittest/main.py
+++ b/Lib/unittest/main.py
@@ -161,9 +161,12 @@ def _initArgParsers(self):
def _getParentArgParser(self):
parser = argparse.ArgumentParser(add_help=False)
+ # Counted, not a constant: the namespace is the TestProgram, whose
+ # verbosity is already 1, so -v still gives 2 and -vv gives 3.
parser.add_argument('-v', '--verbose', dest='verbosity',
- action='store_const', const=2,
- help='Verbose output')
+ action='count', default=1,
+ help='Verbose output, twice to also report '
+ 'the examples of a doctest')
parser.add_argument('-q', '--quiet', dest='verbosity',
action='store_const', const=0,
help='Quiet output')
diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py
index b8ea396..a787686 100644
--- a/Lib/unittest/result.py
+++ b/Lib/unittest/result.py
@@ -37,6 +37,9 @@ class TestResult(object):
_moduleSetUpFailed = False
def __init__(self, stream=None, descriptions=None, verbosity=None):
self.failfast = False
+ # How much the test runner reports: 0 -- quiet, 1 -- progress dots,
+ # 2 -- test names, 3 -- also the examples of a doctest.
+ self.verbosity = 1 if verbosity is None else verbosity
self.failures = []
self.errors = []
self.testsRun = 0
diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py
index 893fcba..f19d7b6 100644
--- a/Lib/unittest/runner.py
+++ b/Lib/unittest/runner.py
@@ -244,6 +244,9 @@ def run(self, test):
result.failfast = self.failfast
result.buffer = self.buffer
result.tb_locals = self.tb_locals
+ # Not left to _makeResult(): a result class is free to filter the
+ # verbosity which its constructor gets.
+ result.verbosity = self.verbosity
with warnings.catch_warnings():
if self.warnings:
# if self.warnings is set, use it to filter all the warnings
diff --git a/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst b/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst
new file mode 100644
index 0000000..e504f9b
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst
@@ -0,0 +1,6 @@
+Doctests run by the :mod:`unittest` test runner now report every example, as
+in verbose mode, if the runner reports more than the test names, i.e. its
+verbosity is 3 or higher. The :option:`!-v` option of :mod:`unittest` can now
+be repeated, so ``python -m unittest -vv`` asks for this. Added also the
+:attr:`~unittest.TestResult.verbosity` attribute of
+:class:`unittest.TestResult`.
diff --git a/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst b/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst
new file mode 100644
index 0000000..3cdac19
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst
@@ -0,0 +1,2 @@
+Running the Python test suite with ``-vv`` now reports every example of a
+doctest. A single ``-v`` reports the test names, as before.
