gh-159799: Require pathlib.[Pure]Path for file paths in the PEG generator (#159800)
The pegen entry points now take `pathlib.Path` objects instead of
strings, and the command-line tools (`python -m pegen`, `keywordgen`,
`first_sets`, `grammar_visualizer`) parse their path arguments into
`Path` objects.
The generators' `generate` and `prepare` take a `pathlib.PurePath`
since they only use the file's name (for the "generated by" comment
in the output) and callers pass names that are not files, such as
`"<string>"`.
diff --git a/Lib/test/test_peg_generator/test_c_generator.py b/Lib/test/test_peg_generator/test_c_generator.py
index 5b48738..3f66db2 100644
--- a/Lib/test/test_peg_generator/test_c_generator.py
+++ b/Lib/test/test_peg_generator/test_c_generator.py
@@ -1,5 +1,6 @@
import io
import unittest
+from pathlib import PurePath
from unittest import mock
from test import test_tools
@@ -33,7 +34,7 @@ def test_rule_types_distinguish_implicit_and_explicit_void_pointer(self):
explicit[void*]: NAME
""")
generator.rules["explicit"].type = "void *"
- start, implicit, explicit = generator.prepare("example.gram").rules
+ start, implicit, explicit = generator.prepare(PurePath("example.gram")).rules
self.assertIsNone(implicit.signature.return_type)
self.assertEqual(explicit.signature.return_type, "void *")
@@ -49,7 +50,7 @@ def test_rule_types_distinguish_implicit_and_explicit_void_pointer(self):
self.assertEqual(call.function, "_PyPegen_lookahead")
generator = self.make_generator("start: &explicit\nexplicit[void*]: NAME\n")
with self.assertRaisesRegex(RuntimeError, "return type is incompatible"):
- generator.prepare("example.gram")
+ generator.prepare(PurePath("example.gram"))
def test_parser_plan_does_not_depend_on_compilation_state(self):
generator = self.make_generator("""
@@ -63,7 +64,7 @@ def test_parser_plan_does_not_depend_on_compilation_state(self):
expr[expr_ty]: expr '+' NAME | NAME
""")
generator.debug = True
- parser = generator.prepare("some/path/example.gram")
+ parser = generator.prepare(PurePath("some/path/example.gram"))
expected = self.emit_parser(parser)
self.assertEqual(generator.file.getvalue(), "")
@@ -101,12 +102,12 @@ def test_repeated_preparation_keeps_variable_names_local(self):
other: name_var=NUMBER name_var=NAME { name_var_1 }
"""
generator = self.make_generator(source)
- parser = generator.prepare("example.gram")
+ parser = generator.prepare(PurePath("example.gram"))
expected = self.emit_parser(parser)
self.assertEqual(self.emit_parser(parser), expected)
- self.assertEqual(generator.prepare("example.gram"), parser)
- other = self.make_generator(source).prepare("example.gram")
+ self.assertEqual(generator.prepare(PurePath("example.gram")), parser)
+ other = self.make_generator(source).prepare(PurePath("example.gram"))
self.assertEqual(other, parser)
self.assertEqual(self.emit_parser(other), expected)
self.assertEqual(expected.count("expr_ty name_var_1;"), 2)
@@ -118,7 +119,7 @@ def test_prepared_prefixes_preserve_reuse_and_repeatability(self):
start: prefix ':' NAME | prefix ':' NUMBER | NAME | prefix '=' NAME
prefix[expr_ty] (memo): NAME
""")
- parser = generator.prepare("example.gram")
+ parser = generator.prepare(PurePath("example.gram"))
start = parser.rules[0]
prefix, = start.prefixes
self.assertEqual(prefix.type, "expr_ty")
@@ -126,7 +127,7 @@ def test_prepared_prefixes_preserve_reuse_and_repeatability(self):
self.assertIn("!p->call_invalid_rules", alt.calls[0].expression())
self.assertIn(prefix.result, alt.calls[0].expression())
self.assertEqual(start.alternatives[3].calls[0].expression(), "prefix_rule(p)")
- self.assertEqual(generator.prepare("example.gram"), parser)
+ self.assertEqual(generator.prepare(PurePath("example.gram")), parser)
expected = self.emit_parser(parser)
generator.rules.clear()
generator.all_rules.clear()
@@ -137,7 +138,7 @@ def test_nullable_prefix_is_not_reused(self):
start: prefix ':' NAME | prefix ':' NUMBER
prefix (memo): NAME?
""")
- start = generator.prepare("example.gram").rules[0]
+ start = generator.prepare(PurePath("example.gram")).rules[0]
self.assertEqual(start.prefixes, ())
for alt in start.alternatives:
self.assertEqual(alt.calls[0].expression(), "prefix_rule(p)")
@@ -148,11 +149,11 @@ def test_invalid_trailer_fails_before_output(self):
start: NAME ENDMARKER
""")
with self.assertRaisesRegex(KeyError, "missing"):
- generator.generate("example.gram")
+ generator.generate(PurePath("example.gram"))
self.assertEqual(generator.file.getvalue(), "")
def test_empty_keyword_tables(self):
- parser = self.make_generator("start: NAME ENDMARKER\n").prepare("example.gram")
+ parser = self.make_generator("start: NAME ENDMARKER\n").prepare(PurePath("example.gram"))
source = self.emit_parser(parser)
self.assertEqual(parser.keyword_groups, ())
@@ -195,9 +196,9 @@ def test_helper_resolution_does_not_depend_on_display_settings(self):
grammar_module, "SIMPLE_STR", simple
):
generator = self.make_generator(source)
- expected = generator.prepare("example.gram").rules
+ expected = generator.prepare(PurePath("example.gram")).rules
with mock.patch.object(grammar_module, "SIMPLE_STR", not simple):
- actual = generator.prepare("example.gram").rules
+ actual = generator.prepare(PurePath("example.gram")).rules
self.assertEqual(len(actual), len(expected))
for old, new in zip(expected, actual):
self.assertEqual(old.signature, new.signature)
@@ -232,7 +233,7 @@ def test_invalid_rule_gating_uses_references(self):
start: {item} {{ _PyPegen_dummy_name(p) }}
invalid_example: NAME
""")
- start = generator.prepare("example.gram").rules[0]
+ start = generator.prepare(PurePath("example.gram")).rules[0]
self.assertEqual(
start.alternatives[0].requires_invalid_rules,
requires_invalid_rules,
@@ -243,7 +244,7 @@ def test_lowering_preserves_legacy_named_call_types(self):
start: Mixed LPAR ENDMARKER
Mixed[expr_ty]: NAME
""")
- start, mixed = generator.prepare("example.gram").rules
+ start, mixed = generator.prepare(PurePath("example.gram")).rules
self.assertEqual(mixed.signature.return_type, "expr_ty")
for call, name in zip(start.alternatives[0].calls, ("Mixed", "LPAR")):
with self.subTest(name=name):
diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py
index 64c8753..e95cae0 100644
--- a/Lib/test/test_peg_generator/test_pegen.py
+++ b/Lib/test/test_peg_generator/test_pegen.py
@@ -3,6 +3,7 @@
import io
import textwrap
import unittest
+from pathlib import PurePath
from test import test_tools
from typing import Dict, Any
@@ -573,7 +574,7 @@ def test_mutually_left_recursive(self) -> None:
self.assertFalse(rules["start"].left_recursive)
self.assertTrue(rules["foo"].left_recursive)
self.assertTrue(rules["bar"].left_recursive)
- genr.generate("<string>")
+ genr.generate(PurePath("<string>"))
ns: Dict[str, Any] = {}
exec(out.getvalue(), ns)
parser_class: Type[Parser] = ns["GeneratedParser"]
@@ -670,7 +671,7 @@ def test_nasty_mutually_left_recursive(self) -> None:
grammar: Grammar = parse_string(grammar_source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
- genr.generate("<string>")
+ genr.generate(PurePath("<string>"))
ns: Dict[str, Any] = {}
exec(out.getvalue(), ns)
parser_class = ns["GeneratedParser"]
@@ -902,7 +903,7 @@ def test_unreachable_explicit(self) -> None:
genr = PythonParserGenerator(
grammar, out, unreachable_formatting="This is a test"
)
- genr.generate("<string>")
+ genr.generate(PurePath("<string>"))
self.assertIn("This is a test", out.getvalue())
def test_unreachable_implicit1(self) -> None:
@@ -915,7 +916,7 @@ def test_unreachable_implicit1(self) -> None:
genr = PythonParserGenerator(
grammar, out, unreachable_formatting="This is a test"
)
- genr.generate("<string>")
+ genr.generate(PurePath("<string>"))
self.assertIn("This is a test", out.getvalue())
def test_unreachable_implicit2(self) -> None:
@@ -928,7 +929,7 @@ def test_unreachable_implicit2(self) -> None:
genr = PythonParserGenerator(
grammar, out, unreachable_formatting="This is a test"
)
- genr.generate("<string>")
+ genr.generate(PurePath("<string>"))
self.assertIn("This is a test", out.getvalue())
def test_unreachable_implicit3(self) -> None:
@@ -941,7 +942,7 @@ def test_unreachable_implicit3(self) -> None:
genr = PythonParserGenerator(
grammar, out, unreachable_formatting="This is a test"
)
- genr.generate("<string>")
+ genr.generate(PurePath("<string>"))
self.assertNotIn("This is a test", out.getvalue())
def test_locations_in_alt_action_and_group(self) -> None:
diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py
index 7a82ad8..e275c17 100755
--- a/Tools/peg_generator/pegen/__main__.py
+++ b/Tools/peg_generator/pegen/__main__.py
@@ -6,6 +6,7 @@
"""
import argparse
+import pathlib
import sys
import time
import token
@@ -87,10 +88,15 @@ def generate_python_code(
c_parser = subparsers.add_parser("c", help="Generate C code for inclusion into CPython")
c_parser.set_defaults(func=generate_c_code)
-c_parser.add_argument("grammar_filename", help="Grammar description")
-c_parser.add_argument("tokens_filename", help="Tokens description")
+c_parser.add_argument("grammar_filename", type=pathlib.Path, help="Grammar description")
+c_parser.add_argument("tokens_filename", type=pathlib.Path, help="Tokens description")
c_parser.add_argument(
- "-o", "--output", metavar="OUT", default="parse.c", help="Where to write the generated parser"
+ "-o",
+ "--output",
+ metavar="OUT",
+ type=pathlib.Path,
+ default=pathlib.Path("parse.c"),
+ help="Where to write the generated parser",
)
c_parser.add_argument(
"--compile-extension",
@@ -111,12 +117,13 @@ def generate_python_code(
help="Generate Python code, needs grammar definition with Python actions",
)
python_parser.set_defaults(func=generate_python_code)
-python_parser.add_argument("grammar_filename", help="Grammar description")
+python_parser.add_argument("grammar_filename", type=pathlib.Path, help="Grammar description")
python_parser.add_argument(
"-o",
"--output",
metavar="OUT",
- default="parse.py",
+ type=pathlib.Path,
+ default=pathlib.Path("parse.py"),
help="Where to write the generated parser",
)
python_parser.add_argument(
diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py
index eb556e5..684492c 100644
--- a/Tools/peg_generator/pegen/build.py
+++ b/Tools/peg_generator/pegen/build.py
@@ -70,7 +70,7 @@ def fixup_build_ext(cmd: Incomplete) -> None:
def compile_c_extension(
- generated_source_path: str,
+ generated_source_path: pathlib.Path,
build_dir: str | None = None,
verbose: bool = False,
keep_asserts: bool = True,
@@ -101,8 +101,7 @@ def compile_c_extension(
if verbose:
setuptools.logging.set_threshold(logging.DEBUG)
- source_file_path = pathlib.Path(generated_source_path)
- extension_name = source_file_path.stem
+ extension_name = generated_source_path.stem
extra_compile_args = get_extra_flags("CFLAGS", "PY_CFLAGS_NODIST")
extra_compile_args.append("-DPy_BUILD_CORE_MODULE")
# Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c
@@ -122,39 +121,39 @@ def compile_c_extension(
extra_link_args.append("-fno-lto")
common_sources = [
- str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"),
- str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "layout.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "decoder.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "api.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "reader.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "pegen.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "pegen_errors.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "action_helpers.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "string_parser.c"),
- str(MOD_DIR.parent / "peg_extension" / "peg_extension.c"),
+ MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c",
+ MOD_DIR.parent.parent.parent / "Python" / "asdl.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "layout.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "decoder.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "api.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "reader.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "pegen.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "pegen_errors.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "action_helpers.c",
+ MOD_DIR.parent.parent.parent / "Parser" / "string_parser.c",
+ MOD_DIR.parent / "peg_extension" / "peg_extension.c",
]
include_dirs = [
- str(MOD_DIR.parent.parent.parent / "Include" / "internal"),
- str(MOD_DIR.parent.parent.parent / "Include" / "internal" / "mimalloc"),
- str(MOD_DIR.parent.parent.parent / "Parser"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "lexer"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer"),
+ MOD_DIR.parent.parent.parent / "Include" / "internal",
+ MOD_DIR.parent.parent.parent / "Include" / "internal" / "mimalloc",
+ MOD_DIR.parent.parent.parent / "Parser",
+ MOD_DIR.parent.parent.parent / "Parser" / "lexer",
+ MOD_DIR.parent.parent.parent / "Parser" / "tokenizer",
]
- library_dirs: list[str] = []
+ library_dirs: list[pathlib.Path] = []
if sys.platform == "win32":
# HACK: The location of pyconfig.h has moved within our build, and
# setuptools hasn't updated for it yet. So add the path manually for now
- include_dirs.append(str(pathlib.Path(sysconfig.get_config_h_filename()).parent))
+ include_dirs.append(pathlib.Path(sysconfig.get_config_h_filename()).parent)
if sysconfig.is_python_build():
# HACK: Our output directory for free-threaded builds has moved, and so
# tests running in-tree require our sys.executable directory for libs
- library_dirs.append(str(pathlib.Path(sys.executable).parent))
+ library_dirs.append(pathlib.Path(sys.executable).parent)
sources = [generated_source_path]
extension = Extension(
extension_name,
@@ -166,9 +165,9 @@ def compile_c_extension(
cmd = dist.get_command_obj("build_ext")
assert isinstance(cmd, setuptools.command.build_ext.build_ext)
fixup_build_ext(cmd)
- cmd.build_lib = str(source_file_path.parent)
- cmd.include_dirs = include_dirs
- cmd.library_dirs = library_dirs
+ cmd.build_lib = str(generated_source_path.parent)
+ cmd.include_dirs = [str(d) for d in include_dirs]
+ cmd.library_dirs = [str(d) for d in library_dirs]
if build_dir:
cmd.build_temp = build_dir
cmd.ensure_finalized()
@@ -247,7 +246,7 @@ def compile_c_extension(
def build_parser(
- grammar_file: str, verbose_tokenizer: bool = False, verbose_parser: bool = False
+ grammar_file: pathlib.Path, verbose_tokenizer: bool = False, verbose_parser: bool = False
) -> tuple[Grammar, Parser, Tokenizer]:
with open(grammar_file) as file:
tokenizer = Tokenizer(tokenize.generate_tokens(file.readline), verbose=verbose_tokenizer)
@@ -255,7 +254,7 @@ def build_parser(
grammar = parser.start()
if not grammar:
- raise parser.make_syntax_error(grammar_file)
+ raise parser.make_syntax_error(str(grammar_file))
return grammar, parser, tokenizer
@@ -291,9 +290,9 @@ def generate_token_definitions(tokens: IO[str]) -> TokenDefinitions:
def build_c_generator(
grammar: Grammar,
- grammar_file: str,
- tokens_file: str,
- output_file: str,
+ grammar_file: pathlib.Path,
+ tokens_file: pathlib.Path,
+ output_file: pathlib.Path,
compile_extension: bool = False,
verbose_c_extension: bool = False,
keep_asserts_in_extension: bool = True,
@@ -320,8 +319,8 @@ def build_c_generator(
def build_python_generator(
grammar: Grammar,
- grammar_file: str,
- output_file: str,
+ grammar_file: pathlib.Path,
+ output_file: pathlib.Path,
skip_actions: bool = False,
) -> ParserGenerator:
with open(output_file, "w") as file:
@@ -331,9 +330,9 @@ def build_python_generator(
def build_c_parser_and_generator(
- grammar_file: str,
- tokens_file: str,
- output_file: str,
+ grammar_file: pathlib.Path,
+ tokens_file: pathlib.Path,
+ output_file: pathlib.Path,
compile_extension: bool = False,
verbose_tokenizer: bool = False,
verbose_parser: bool = False,
@@ -375,8 +374,8 @@ def build_c_parser_and_generator(
def build_python_parser_and_generator(
- grammar_file: str,
- output_file: str,
+ grammar_file: pathlib.Path,
+ output_file: pathlib.Path,
verbose_tokenizer: bool = False,
verbose_parser: bool = False,
skip_actions: bool = False,
diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index b2978d3..d1f8781 100644
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -1,6 +1,6 @@
"""Prepare a complete C parser description and emit it."""
-import os.path
+import pathlib
from typing import IO
from pegen import grammar
@@ -76,11 +76,11 @@ def __init__(
self.debug = debug
self.skip_actions = skip_actions
- def generate(self, filename: str) -> None:
+ def generate(self, filename: pathlib.PurePath) -> None:
parser = self.prepare(filename)
CParserEmitter(parser, self.file).emit()
- def prepare(self, filename: str) -> CParser:
+ def prepare(self, filename: pathlib.PurePath) -> CParser:
self.collect_rules()
lowerer = self.callmakervisitor.make_lowerer()
rules = tuple(
@@ -93,7 +93,7 @@ def prepare(self, filename: str) -> CParser:
if subheader := self.grammar.metas.get("subheader", ""):
headers.append(subheader)
return CParser(
- source_name=os.path.basename(filename),
+ source_name=filename.name,
headers=tuple(headers),
keyword_groups=self._prepare_keywords(),
soft_keywords=tuple(sorted(self.soft_keywords)),
diff --git a/Tools/peg_generator/pegen/first_sets.py b/Tools/peg_generator/pegen/first_sets.py
index f6b83c0..9980f76 100755
--- a/Tools/peg_generator/pegen/first_sets.py
+++ b/Tools/peg_generator/pegen/first_sets.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3.8
import argparse
+import pathlib
import pprint
import sys
@@ -28,7 +29,7 @@
prog="calculate_first_sets",
description="Calculate the first sets of a grammar",
)
-argparser.add_argument("grammar_file", help="The grammar file")
+argparser.add_argument("grammar_file", type=pathlib.Path, help="The grammar file")
class FirstSetCalculator(GrammarVisitor):
diff --git a/Tools/peg_generator/pegen/grammar_visualizer.py b/Tools/peg_generator/pegen/grammar_visualizer.py
index eadb698..77a5bc7 100644
--- a/Tools/peg_generator/pegen/grammar_visualizer.py
+++ b/Tools/peg_generator/pegen/grammar_visualizer.py
@@ -1,4 +1,5 @@
import argparse
+import pathlib
import sys
from collections.abc import Callable, Iterator
from typing import Any
@@ -9,7 +10,7 @@
argparser = argparse.ArgumentParser(
prog="pegen", description="Pretty print the AST for a given PEG grammar"
)
-argparser.add_argument("filename", help="Grammar description")
+argparser.add_argument("filename", type=pathlib.Path, help="Grammar description")
class ASTGrammarPrinter:
diff --git a/Tools/peg_generator/pegen/keywordgen.py b/Tools/peg_generator/pegen/keywordgen.py
index 52611ea..1ded1d3 100644
--- a/Tools/peg_generator/pegen/keywordgen.py
+++ b/Tools/peg_generator/pegen/keywordgen.py
@@ -1,6 +1,7 @@
"""Generate Lib/keyword.py from the Grammar and Tokens files using pgen"""
import argparse
+import pathlib
from .build import build_parser, generate_token_definitions
from .c_generator import CParserGenerator
@@ -41,13 +42,14 @@ def main() -> None:
description="Generate the Lib/keywords.py file from the grammar."
)
parser.add_argument(
- "grammar", help="The file with the grammar definition in PEG format"
+ "grammar", type=pathlib.Path, help="The file with the grammar definition in PEG format"
)
parser.add_argument(
- "tokens_file", help="The file with the token definitions"
+ "tokens_file", type=pathlib.Path, help="The file with the token definitions"
)
parser.add_argument(
"keyword_file",
+ type=pathlib.Path,
help="The path to write the keyword definitions",
)
args = parser.parse_args()
diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py
index f7783f1..18736be 100644
--- a/Tools/peg_generator/pegen/parser_generator.py
+++ b/Tools/peg_generator/pegen/parser_generator.py
@@ -1,5 +1,6 @@
import ast
import contextlib
+import pathlib
import re
import sys
from abc import abstractmethod
@@ -134,7 +135,7 @@ def local_variable_names(self) -> list[str]:
return self._local_variable_stack[-1]
@abstractmethod
- def generate(self, filename: str) -> None:
+ def generate(self, filename: pathlib.PurePath) -> None:
raise NotImplementedError
@contextlib.contextmanager
diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py
index e69ea4b..cf3eb86 100644
--- a/Tools/peg_generator/pegen/python_generator.py
+++ b/Tools/peg_generator/pegen/python_generator.py
@@ -1,4 +1,4 @@
-import os.path
+import pathlib
import token
from collections.abc import Callable, Sequence
from typing import IO, Any
@@ -232,12 +232,11 @@ def __init__(
"end_lineno=end_lineno, end_col_offset=end_col_offset"
)
- def generate(self, filename: str) -> None:
+ def generate(self, filename: pathlib.PurePath) -> None:
self.collect_rules()
header = self.grammar.metas.get("header", MODULE_PREFIX)
if header is not None:
- basename = os.path.basename(filename)
- self.print(header.rstrip("\n").format(filename=basename))
+ self.print(header.rstrip("\n").format(filename=filename.name))
subheader = self.grammar.metas.get("subheader", "")
if subheader:
self.print(subheader)
diff --git a/Tools/peg_generator/pegen/testutil.py b/Tools/peg_generator/pegen/testutil.py
index 46f6c7f..a09e0b9 100644
--- a/Tools/peg_generator/pegen/testutil.py
+++ b/Tools/peg_generator/pegen/testutil.py
@@ -1,11 +1,11 @@
import importlib.util
import io
import os
-import pathlib
import sys
import textwrap
import token
import tokenize
+from pathlib import Path, PurePath
from typing import IO, Any, Final, cast
from pegen.build import compile_c_extension
@@ -27,7 +27,7 @@ def generate_parser(grammar: Grammar) -> type[Parser]:
# Generate a parser.
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
- genr.generate("<string>")
+ genr.generate(PurePath("<string>"))
# Load the generated parser class.
ns: dict[str, Any] = {}
@@ -78,13 +78,13 @@ def import_file(full_name: str, path: str) -> Any:
def generate_c_parser_source(grammar: Grammar) -> str:
out = io.StringIO()
genr = CParserGenerator(grammar, ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, out)
- genr.generate("<string>")
+ genr.generate(PurePath("<string>"))
return out.getvalue()
def generate_parser_c_extension(
grammar: Grammar,
- path: pathlib.PurePath,
+ path: Path,
debug: bool = False,
library_dir: str | None = None,
) -> Any:
@@ -103,9 +103,9 @@ def generate_parser_c_extension(
genr = CParserGenerator(
grammar, ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, file, debug=debug
)
- genr.generate("parse.c")
+ genr.generate(PurePath("parse.c"))
compile_c_extension(
- str(source),
+ source,
build_dir=str(path),
# Significant test_peg_generator speedups
disable_optimization=True,
