gh-69371: Fix pydoc for modules whose path contains undecodable bytes (GH-155239)
"python -m pydoc -w" and the pydoc HTTP server failed with
UnicodeEncodeError. The file URL is now created with
urllib.request.pathname2url(), which percent-encodes the path using the
filesystem encoding, and characters unencodable in the generated UTF-8
page are escaped with backslashes.
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
index 3cba08b..6f3dab2 100644
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -71,7 +71,6 @@ class or function within a module or module in a package. If the
import textwrap
import time
import tokenize
-import urllib.parse
import warnings
from annotationlib import Format
from collections import deque
@@ -795,7 +794,8 @@ def docmodule(self, object, name=None, mod=None, *ignored):
head = linkedname
try:
path = inspect.getabsfile(object)
- url = urllib.parse.quote(path)
+ import urllib.request
+ url = urllib.request.pathname2url(path)
filelink = self.filelink(url, path)
except TypeError:
filelink = '(built-in)'
@@ -1790,7 +1790,8 @@ def writedoc(thing, forceload=0):
"""Write HTML documentation to a file in the current directory."""
object, name = resolve(thing, forceload)
page = html.page(describe(object), html.document(object, name))
- with open(name + '.html', 'w', encoding='utf-8') as file:
+ with open(name + '.html', 'w', encoding='utf-8',
+ errors='backslashreplace') as file:
file.write(page)
print('wrote', name + '.html')
@@ -2387,7 +2388,7 @@ def do_GET(self):
self.send_header('Content-Type', '%s; charset=UTF-8' % content_type)
self.end_headers()
self.wfile.write(self.urlhandler(
- self.path, content_type).encode('utf-8'))
+ self.path, content_type).encode('utf-8', 'backslashreplace'))
def log_message(self, *args):
# Don't log messages.
diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py
index 34d30f5..006fbc8 100644
--- a/Lib/test/test_pydoc/test_pydoc.py
+++ b/Lib/test/test_pydoc/test_pydoc.py
@@ -19,6 +19,7 @@
import unittest
import unittest.mock
import urllib.parse
+import urllib.request
import xml.etree
import xml.etree.ElementTree
import textwrap
@@ -429,7 +430,7 @@ def test_html_doc(self):
expected_lines = [line.strip() for line in expected_lines if line]
self.assertEqual(text_lines, expected_lines)
mod_file = inspect.getabsfile(pydoc_mod)
- mod_url = urllib.parse.quote(mod_file)
+ mod_url = urllib.request.pathname2url(mod_file)
self.assertIn(mod_url, result)
self.assertIn(mod_file, result)
self.assertIn(doc_loc, result)
@@ -1024,6 +1025,31 @@ def test_synopsis_sourceless_empty_doc(self):
synopsis_cached = pydoc.synopsis(cached_path, {})
self.assertIsNone(synopsis_cached)
+ @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE,
+ 'requires undecodable file names')
+ def test_html_doc_undecodable_path(self):
+ # gh-69371: the path of the module is not encodable in UTF-8.
+ with os_helper.temp_cwd() as test_dir:
+ subdir = os.path.join(os.fsencode(test_dir),
+ os_helper.TESTFN_UNDECODABLE)
+ try:
+ os.mkdir(subdir)
+ except OSError:
+ self.skipTest('undecodable paths are not supported')
+ with open(os.path.join(subdir, b'undecodable_mod.py'), 'w') as f:
+ f.write('"""Module docstring."""\n')
+ with import_helper.DirsOnSysPath(os.fsdecode(subdir)):
+ mod = import_helper.import_fresh_module('undecodable_mod')
+ doc = pydoc.HTMLDoc().docmodule(mod)
+ with captured_stdout():
+ pydoc.writedoc(mod)
+ # The link contains the percent-encoded path...
+ path = inspect.getabsfile(mod)
+ self.assertIn(urllib.request.pathname2url(path), doc)
+ # ...and the page can be written and served as UTF-8.
+ with open('undecodable_mod.html', encoding='utf-8') as f:
+ self.assertIn('undecodable_mod', f.read())
+
def test_splitdoc_with_description(self):
example_string = "I Am A Doc\n\n\nHere is my description"
self.assertEqual(pydoc.splitdoc(example_string),
diff --git a/Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst b/Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst
new file mode 100644
index 0000000..4d8f866
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst
@@ -0,0 +1,5 @@
+Fix :mod:`pydoc` for modules whose path contains undecodable bytes.
+:func:`!pydoc.writedoc` and the pydoc HTTP server no longer fail with
+:exc:`UnicodeEncodeError`: the file URL is now percent-encoded using the
+filesystem encoding, and characters unencodable in the generated HTML page
+are escaped with backslashes.