blob: d182e5e7764e46e6078da4f899d220d2bd99cdf0 [file] [log] [blame]
#!/usr/bin/env python3
#
# Argument Clinic
# Copyright 2012-2013 by Larry Hastings.
# Licensed to the PSF under a contributor agreement.
#
import abc
import ast
import builtins as bltns
import collections
import contextlib
import copy
import cpp
import enum
import functools
import hashlib
import inspect
import io
import itertools
import os
import pprint
import re
import shlex
import string
import sys
import textwrap
import traceback
from collections.abc import Callable
from types import FunctionType, NoneType
from typing import Any, Final, NamedTuple, NoReturn, Literal, overload
# TODO:
#
# soon:
#
# * allow mixing any two of {positional-only, positional-or-keyword,
# keyword-only}
# * dict constructor uses positional-only and keyword-only
# * max and min use positional only with an optional group
# and keyword-only
#
version = '1'
NO_VARARG = "PY_SSIZE_T_MAX"
CLINIC_PREFIX = "__clinic_"
CLINIC_PREFIXED_ARGS = {
"_keywords",
"_parser",
"args",
"argsbuf",
"fastargs",
"kwargs",
"kwnames",
"nargs",
"noptargs",
"return_value",
}
class Sentinels(enum.Enum):
unspecified = "unspecified"
unknown = "unknown"
def __repr__(self) -> str:
return f"<{self.value.capitalize()}>"
unspecified: Final = Sentinels.unspecified
unknown: Final = Sentinels.unknown
# This one needs to be a distinct class, unlike the other two
class Null:
def __repr__(self) -> str:
return '<Null>'
NULL = Null()
sig_end_marker = '--'
Appender = Callable[[str], None]
Outputter = Callable[[], str]
TemplateDict = dict[str, str]
class _TextAccumulator(NamedTuple):
text: list[str]
append: Appender
output: Outputter
def _text_accumulator() -> _TextAccumulator:
text: list[str] = []
def output():
s = ''.join(text)
text.clear()
return s
return _TextAccumulator(text, text.append, output)
class TextAccumulator(NamedTuple):
append: Appender
output: Outputter
def text_accumulator() -> TextAccumulator:
"""
Creates a simple text accumulator / joiner.
Returns a pair of callables:
append, output
"append" appends a string to the accumulator.
"output" returns the contents of the accumulator
joined together (''.join(accumulator)) and
empties the accumulator.
"""
text, append, output = _text_accumulator()
return TextAccumulator(append, output)
@overload
def warn_or_fail(
*args: object,
fail: Literal[True],
filename: str | None = None,
line_number: int | None = None,
) -> NoReturn: ...
@overload
def warn_or_fail(
*args: object,
fail: Literal[False] = False,
filename: str | None = None,
line_number: int | None = None,
) -> None: ...
def warn_or_fail(
*args: object,
fail: bool = False,
filename: str | None = None,
line_number: int | None = None,
) -> None:
joined = " ".join([str(a) for a in args])
add, output = text_accumulator()
if fail:
add("Error")
else:
add("Warning")
if clinic:
if filename is None:
filename = clinic.filename
if getattr(clinic, 'block_parser', None) and (line_number is None):
line_number = clinic.block_parser.line_number
if filename is not None:
add(' in file "' + filename + '"')
if line_number is not None:
add(" on line " + str(line_number))
add(':\n')
add(joined)
print(output())
if fail:
sys.exit(-1)
def warn(
*args: object,
filename: str | None = None,
line_number: int | None = None,
) -> None:
return warn_or_fail(*args, filename=filename, line_number=line_number, fail=False)
def fail(
*args: object,
filename: str | None = None,
line_number: int | None = None,
) -> NoReturn:
warn_or_fail(*args, filename=filename, line_number=line_number, fail=True)
def quoted_for_c_string(s: str) -> str:
for old, new in (
('\\', '\\\\'), # must be first!
('"', '\\"'),
("'", "\\'"),
):
s = s.replace(old, new)
return s
def c_repr(s: str) -> str:
return '"' + s + '"'
is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match
def is_legal_py_identifier(s: str) -> bool:
return all(is_legal_c_identifier(field) for field in s.split('.'))
# identifiers that are okay in Python but aren't a good idea in C.
# so if they're used Argument Clinic will add "_value" to the end
# of the name in C.
c_keywords = set("""
asm auto break case char const continue default do double
else enum extern float for goto if inline int long
register return short signed sizeof static struct switch
typedef typeof union unsigned void volatile while
""".strip().split())
def ensure_legal_c_identifier(s: str) -> str:
# for now, just complain if what we're given isn't legal
if not is_legal_c_identifier(s):
fail("Illegal C identifier:", s)
# but if we picked a C keyword, pick something else
if s in c_keywords:
return s + "_value"
return s
def rstrip_lines(s: str) -> str:
text, add, output = _text_accumulator()
for line in s.split('\n'):
add(line.rstrip())
add('\n')
text.pop()
return output()
def format_escape(s: str) -> str:
# double up curly-braces, this string will be used
# as part of a format_map() template later
s = s.replace('{', '{{')
s = s.replace('}', '}}')
return s
def linear_format(s: str, **kwargs: str) -> str:
"""
Perform str.format-like substitution, except:
* The strings substituted must be on lines by
themselves. (This line is the "source line".)
* If the substitution text is empty, the source line
is removed in the output.
* If the field is not recognized, the original line
is passed unmodified through to the output.
* If the substitution text is not empty:
* Each line of the substituted text is indented
by the indent of the source line.
* A newline will be added to the end.
"""
add, output = text_accumulator()
for line in s.split('\n'):
indent, curly, trailing = line.partition('{')
if not curly:
add(line)
add('\n')
continue
name, curly, trailing = trailing.partition('}')
if not curly or name not in kwargs:
add(line)
add('\n')
continue
if trailing:
fail("Text found after {" + name + "} block marker! It must be on a line by itself.")
if indent.strip():
fail("Non-whitespace characters found before {" + name + "} block marker! It must be on a line by itself.")
value = kwargs[name]
if not value:
continue
value = textwrap.indent(rstrip_lines(value), indent)
add(value)
add('\n')
return output()[:-1]
def indent_all_lines(s: str, prefix: str) -> str:
"""
Returns 's', with 'prefix' prepended to all lines.
If the last line is empty, prefix is not prepended
to it. (If s is blank, returns s unchanged.)
(textwrap.indent only adds to non-blank lines.)
"""
split = s.split('\n')
last = split.pop()
final = []
for line in split:
final.append(prefix)
final.append(line)
final.append('\n')
if last:
final.append(prefix)
final.append(last)
return ''.join(final)
def suffix_all_lines(s: str, suffix: str) -> str:
"""
Returns 's', with 'suffix' appended to all lines.
If the last line is empty, suffix is not appended
to it. (If s is blank, returns s unchanged.)
"""
split = s.split('\n')
last = split.pop()
final = []
for line in split:
final.append(line)
final.append(suffix)
final.append('\n')
if last:
final.append(last)
final.append(suffix)
return ''.join(final)
def version_splitter(s: str) -> tuple[int, ...]:
"""Splits a version string into a tuple of integers.
The following ASCII characters are allowed, and employ
the following conversions:
a -> -3
b -> -2
c -> -1
(This permits Python-style version strings such as "1.4b3".)
"""
version: list[int] = []
accumulator: list[str] = []
def flush() -> None:
if not accumulator:
raise ValueError('Unsupported version string: ' + repr(s))
version.append(int(''.join(accumulator)))
accumulator.clear()
for c in s:
if c.isdigit():
accumulator.append(c)
elif c == '.':
flush()
elif c in 'abc':
flush()
version.append('abc'.index(c) - 3)
else:
raise ValueError('Illegal character ' + repr(c) + ' in version string ' + repr(s))
flush()
return tuple(version)
def version_comparitor(version1: str, version2: str) -> Literal[-1, 0, 1]:
iterator = itertools.zip_longest(version_splitter(version1), version_splitter(version2), fillvalue=0)
for i, (a, b) in enumerate(iterator):
if a < b:
return -1
if a > b:
return 1
return 0
class CRenderData:
def __init__(self):
# The C statements to declare variables.
# Should be full lines with \n eol characters.
self.declarations = []
# The C statements required to initialize the variables before the parse call.
# Should be full lines with \n eol characters.
self.initializers = []
# The C statements needed to dynamically modify the values
# parsed by the parse call, before calling the impl.
self.modifications = []
# The entries for the "keywords" array for PyArg_ParseTuple.
# Should be individual strings representing the names.
self.keywords = []
# The "format units" for PyArg_ParseTuple.
# Should be individual strings that will get
self.format_units = []
# The varargs arguments for PyArg_ParseTuple.
self.parse_arguments = []
# The parameter declarations for the impl function.
self.impl_parameters = []
# The arguments to the impl function at the time it's called.
self.impl_arguments = []
# For return converters: the name of the variable that
# should receive the value returned by the impl.
self.return_value = "return_value"
# For return converters: the code to convert the return
# value from the parse function. This is also where
# you should check the _return_value for errors, and
# "goto exit" if there are any.
self.return_conversion = []
self.converter_retval = "_return_value"
# The C statements required to do some operations
# after the end of parsing but before cleaning up.
# These operations may be, for example, memory deallocations which
# can only be done without any error happening during argument parsing.
self.post_parsing = []
# The C statements required to clean up after the impl call.
self.cleanup = []
class FormatCounterFormatter(string.Formatter):
"""
This counts how many instances of each formatter
"replacement string" appear in the format string.
e.g. after evaluating "string {a}, {b}, {c}, {a}"
the counts dict would now look like
{'a': 2, 'b': 1, 'c': 1}
"""
def __init__(self):
self.counts = collections.Counter()
def get_value(self, key, args, kwargs):
self.counts[key] += 1
return ''
class Language(metaclass=abc.ABCMeta):
start_line = ""
body_prefix = ""
stop_line = ""
checksum_line = ""
def __init__(self, filename):
pass
@abc.abstractmethod
def render(self, clinic, signatures):
pass
def parse_line(self, line):
pass
def validate(self):
def assert_only_one(attr, *additional_fields):
"""
Ensures that the string found at getattr(self, attr)
contains exactly one formatter replacement string for
each valid field. The list of valid fields is
['dsl_name'] extended by additional_fields.
e.g.
self.fmt = "{dsl_name} {a} {b}"
# this passes
self.assert_only_one('fmt', 'a', 'b')
# this fails, the format string has a {b} in it
self.assert_only_one('fmt', 'a')
# this fails, the format string doesn't have a {c} in it
self.assert_only_one('fmt', 'a', 'b', 'c')
# this fails, the format string has two {a}s in it,
# it must contain exactly one
self.fmt2 = '{dsl_name} {a} {a}'
self.assert_only_one('fmt2', 'a')
"""
fields = ['dsl_name']
fields.extend(additional_fields)
line = getattr(self, attr)
fcf = FormatCounterFormatter()
fcf.format(line)
def local_fail(should_be_there_but_isnt):
if should_be_there_but_isnt:
fail("{} {} must contain {{{}}} exactly once!".format(
self.__class__.__name__, attr, name))
else:
fail("{} {} must not contain {{{}}}!".format(
self.__class__.__name__, attr, name))
for name, count in fcf.counts.items():
if name in fields:
if count > 1:
local_fail(True)
else:
local_fail(False)
for name in fields:
if fcf.counts.get(name) != 1:
local_fail(True)
assert_only_one('start_line')
assert_only_one('stop_line')
field = "arguments" if "{arguments}" in self.checksum_line else "checksum"
assert_only_one('checksum_line', field)
class PythonLanguage(Language):
language = 'Python'
start_line = "#/*[{dsl_name} input]"
body_prefix = "#"
stop_line = "#[{dsl_name} start generated code]*/"
checksum_line = "#/*[{dsl_name} end generated code: {arguments}]*/"
def permute_left_option_groups(l):
"""
Given [1, 2, 3], should yield:
()
(3,)
(2, 3)
(1, 2, 3)
"""
yield tuple()
accumulator = []
for group in reversed(l):
accumulator = list(group) + accumulator
yield tuple(accumulator)
def permute_right_option_groups(l):
"""
Given [1, 2, 3], should yield:
()
(1,)
(1, 2)
(1, 2, 3)
"""
yield tuple()
accumulator = []
for group in l:
accumulator.extend(group)
yield tuple(accumulator)
def permute_optional_groups(left, required, right):
"""
Generator function that computes the set of acceptable
argument lists for the provided iterables of
argument groups. (Actually it generates a tuple of tuples.)
Algorithm: prefer left options over right options.
If required is empty, left must also be empty.
"""
required = tuple(required)
if not required:
if left:
raise ValueError("required is empty but left is not")
accumulator = []
counts = set()
for r in permute_right_option_groups(right):
for l in permute_left_option_groups(left):
t = l + required + r
if len(t) in counts:
continue
counts.add(len(t))
accumulator.append(t)
accumulator.sort(key=len)
return tuple(accumulator)
def strip_leading_and_trailing_blank_lines(s):
lines = s.rstrip().split('\n')
while lines:
line = lines[0]
if line.strip():
break
del lines[0]
return '\n'.join(lines)
@functools.lru_cache()
def normalize_snippet(s, *, indent=0):
"""
Reformats s:
* removes leading and trailing blank lines
* ensures that it does not end with a newline
* dedents so the first nonwhite character on any line is at column "indent"
"""
s = strip_leading_and_trailing_blank_lines(s)
s = textwrap.dedent(s)
if indent:
s = textwrap.indent(s, ' ' * indent)
return s
def declare_parser(f, *, hasformat=False):
"""
Generates the code template for a static local PyArg_Parser variable,
with an initializer. For core code (incl. builtin modules) the
kwtuple field is also statically initialized. Otherwise
it is initialized at runtime.
"""
if hasformat:
fname = ''
format_ = '.format = "{format_units}:{name}",'
else:
fname = '.fname = "{name}",'
format_ = ''
num_keywords = len([
p for p in f.parameters.values()
if not p.is_positional_only() and not p.is_vararg()
])
if num_keywords == 0:
declarations = """
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
# define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty)
#else
# define KWTUPLE NULL
#endif
"""
else:
declarations = """
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS %d
static struct {{
PyGC_Head _this_is_not_used;
PyObject_VAR_HEAD
PyObject *ob_item[NUM_KEYWORDS];
}} _kwtuple = {{
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = {{ {keywords_py} }},
}};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
#else // !Py_BUILD_CORE
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
""" % num_keywords
declarations += """
static const char * const _keywords[] = {{{keywords_c} NULL}};
static _PyArg_Parser _parser = {{
.keywords = _keywords,
%s
.kwtuple = KWTUPLE,
}};
#undef KWTUPLE
""" % (format_ or fname)
return normalize_snippet(declarations)
def wrap_declarations(text, length=78):
"""
A simple-minded text wrapper for C function declarations.
It views a declaration line as looking like this:
xxxxxxxx(xxxxxxxxx,xxxxxxxxx)
If called with length=30, it would wrap that line into
xxxxxxxx(xxxxxxxxx,
xxxxxxxxx)
(If the declaration has zero or one parameters, this
function won't wrap it.)
If this doesn't work properly, it's probably better to
start from scratch with a more sophisticated algorithm,
rather than try and improve/debug this dumb little function.
"""
lines = []
for line in text.split('\n'):
prefix, _, after_l_paren = line.partition('(')
if not after_l_paren:
lines.append(line)
continue
parameters, _, after_r_paren = after_l_paren.partition(')')
if not _:
lines.append(line)
continue
if ',' not in parameters:
lines.append(line)
continue
parameters = [x.strip() + ", " for x in parameters.split(',')]
prefix += "("
if len(prefix) < length:
spaces = " " * len(prefix)
else:
spaces = " " * 4
while parameters:
line = prefix
first = True
while parameters:
if (not first and
(len(line) + len(parameters[0]) > length)):
break
line += parameters.pop(0)
first = False
if not parameters:
line = line.rstrip(", ") + ")" + after_r_paren
lines.append(line.rstrip())
prefix = spaces
return "\n".join(lines)
class CLanguage(Language):
body_prefix = "#"
language = 'C'
start_line = "/*[{dsl_name} input]"
body_prefix = ""
stop_line = "[{dsl_name} start generated code]*/"
checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/"
def __init__(self, filename):
super().__init__(filename)
self.cpp = cpp.Monitor(filename)
self.cpp.fail = fail
def parse_line(self, line):
self.cpp.writeline(line)
def render(self, clinic, signatures):
function = None
for o in signatures:
if isinstance(o, Function):
if function:
fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o))
function = o
return self.render_function(clinic, function)
def docstring_for_c_string(self, f):
if re.search(r'[^\x00-\x7F]', f.docstring):
warn("Non-ascii character appear in docstring.")
text, add, output = _text_accumulator()
# turn docstring into a properly quoted C string
for line in f.docstring.split('\n'):
add('"')
add(quoted_for_c_string(line))
add('\\n"\n')
if text[-2] == sig_end_marker:
# If we only have a signature, add the blank line that the
# __text_signature__ getter expects to be there.
add('"\\n"')
else:
text.pop()
add('"')
return ''.join(text)
def output_templates(self, f):
parameters = list(f.parameters.values())
assert parameters
assert isinstance(parameters[0].converter, self_converter)
del parameters[0]
requires_defining_class = False
if parameters and isinstance(parameters[0].converter, defining_class_converter):
requires_defining_class = True
del parameters[0]
converters = [p.converter for p in parameters]
has_option_groups = parameters and (parameters[0].group or parameters[-1].group)
default_return_converter = (not f.return_converter or
f.return_converter.type == 'PyObject *')
new_or_init = f.kind in (METHOD_NEW, METHOD_INIT)
vararg = NO_VARARG
pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0
for i, p in enumerate(parameters, 1):
if p.is_keyword_only():
assert not p.is_positional_only()
if not p.is_optional():
min_kw_only = i - max_pos
elif p.is_vararg():
if vararg != NO_VARARG:
fail("Too many var args")
pseudo_args += 1
vararg = i - 1
else:
if vararg == NO_VARARG:
max_pos = i
if p.is_positional_only():
pos_only = i
if not p.is_optional():
min_pos = i
meth_o = (len(parameters) == 1 and
parameters[0].is_positional_only() and
not converters[0].is_optional() and
not requires_defining_class and
not new_or_init)
# we have to set these things before we're done:
#
# docstring_prototype
# docstring_definition
# impl_prototype
# methoddef_define
# parser_prototype
# parser_definition
# impl_definition
# cpp_if
# cpp_endif
# methoddef_ifndef
return_value_declaration = "PyObject *return_value = NULL;"
methoddef_define = normalize_snippet("""
#define {methoddef_name} \\
{{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}},
""")
if new_or_init and not f.docstring:
docstring_prototype = docstring_definition = ''
else:
docstring_prototype = normalize_snippet("""
PyDoc_VAR({c_basename}__doc__);
""")
docstring_definition = normalize_snippet("""
PyDoc_STRVAR({c_basename}__doc__,
{docstring});
""")
impl_definition = normalize_snippet("""
static {impl_return_type}
{c_basename}_impl({impl_parameters})
""")
impl_prototype = parser_prototype = parser_definition = None
parser_prototype_keyword = normalize_snippet("""
static PyObject *
{c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs)
""")
parser_prototype_varargs = normalize_snippet("""
static PyObject *
{c_basename}({self_type}{self_name}, PyObject *args)
""")
parser_prototype_fastcall = normalize_snippet("""
static PyObject *
{c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs)
""")
parser_prototype_fastcall_keywords = normalize_snippet("""
static PyObject *
{c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
""")
parser_prototype_def_class = normalize_snippet("""
static PyObject *
{c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
""")
# parser_body_fields remembers the fields passed in to the
# previous call to parser_body. this is used for an awful hack.
parser_body_fields = ()
def parser_body(prototype, *fields, declarations=''):
nonlocal parser_body_fields
add, output = text_accumulator()
add(prototype)
parser_body_fields = fields
fields = list(fields)
fields.insert(0, normalize_snippet("""
{{
{return_value_declaration}
{parser_declarations}
{declarations}
{initializers}
""") + "\n")
# just imagine--your code is here in the middle
fields.append(normalize_snippet("""
{modifications}
{return_value} = {c_basename}_impl({impl_arguments});
{return_conversion}
{post_parsing}
{exit_label}
{cleanup}
return return_value;
}}
"""))
for field in fields:
add('\n')
add(field)
return linear_format(output(), parser_declarations=declarations)
if not parameters:
if not requires_defining_class:
# no parameters, METH_NOARGS
flags = "METH_NOARGS"
parser_prototype = normalize_snippet("""
static PyObject *
{c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored))
""")
parser_code = []
else:
assert not new_or_init
flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS"
parser_prototype = parser_prototype_def_class
return_error = ('return NULL;' if default_return_converter
else 'goto exit;')
parser_code = [normalize_snippet("""
if (nargs) {{
PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments");
%s
}}
""" % return_error, indent=4)]
if default_return_converter:
parser_definition = '\n'.join([
parser_prototype,
'{{',
*parser_code,
' return {c_basename}_impl({impl_arguments});',
'}}'])
else:
parser_definition = parser_body(parser_prototype, *parser_code)
elif meth_o:
flags = "METH_O"
if (isinstance(converters[0], object_converter) and
converters[0].format_unit == 'O'):
meth_o_prototype = normalize_snippet("""
static PyObject *
{c_basename}({impl_parameters})
""")
if default_return_converter:
# maps perfectly to METH_O, doesn't need a return converter.
# so we skip making a parse function
# and call directly into the impl function.
impl_prototype = parser_prototype = parser_definition = ''
impl_definition = meth_o_prototype
else:
# SLIGHT HACK
# use impl_parameters for the parser here!
parser_prototype = meth_o_prototype
parser_definition = parser_body(parser_prototype)
else:
argname = 'arg'
if parameters[0].name == argname:
argname += '_'
parser_prototype = normalize_snippet("""
static PyObject *
{c_basename}({self_type}{self_name}, PyObject *%s)
""" % argname)
displayname = parameters[0].get_displayname(0)
parsearg = converters[0].parse_arg(argname, displayname)
if parsearg is None:
parsearg = """
if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{
goto exit;
}}
""" % argname
parser_definition = parser_body(parser_prototype,
normalize_snippet(parsearg, indent=4))
elif has_option_groups:
# positional parameters with option groups
# (we have to generate lots of PyArg_ParseTuple calls
# in a big switch statement)
flags = "METH_VARARGS"
parser_prototype = parser_prototype_varargs
parser_definition = parser_body(parser_prototype, ' {option_group_parsing}')
elif not requires_defining_class and pos_only == len(parameters) - pseudo_args:
if not new_or_init:
# positional-only, but no option groups
# we only need one call to _PyArg_ParseStack
flags = "METH_FASTCALL"
parser_prototype = parser_prototype_fastcall
nargs = 'nargs'
argname_fmt = 'args[%d]'
else:
# positional-only, but no option groups
# we only need one call to PyArg_ParseTuple
flags = "METH_VARARGS"
parser_prototype = parser_prototype_varargs
nargs = 'PyTuple_GET_SIZE(args)'
argname_fmt = 'PyTuple_GET_ITEM(args, %d)'
left_args = f"{nargs} - {max_pos}"
max_args = NO_VARARG if (vararg != NO_VARARG) else max_pos
parser_code = [normalize_snippet("""
if (!_PyArg_CheckPositional("{name}", %s, %d, %s)) {{
goto exit;
}}
""" % (nargs, min_pos, max_args), indent=4)]
has_optional = False
for i, p in enumerate(parameters):
displayname = p.get_displayname(i+1)
argname = argname_fmt % i
if p.is_vararg():
if not new_or_init:
parser_code.append(normalize_snippet("""
%s = PyTuple_New(%s);
if (!%s) {{
goto exit;
}}
for (Py_ssize_t i = 0; i < %s; ++i) {{
PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i]));
}}
""" % (
p.converter.parser_name,
left_args,
p.converter.parser_name,
left_args,
p.converter.parser_name,
max_pos
), indent=4))
else:
parser_code.append(normalize_snippet("""
%s = PyTuple_GetSlice(%d, -1);
""" % (
p.converter.parser_name,
max_pos
), indent=4))
continue
parsearg = p.converter.parse_arg(argname, displayname)
if parsearg is None:
#print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr)
parser_code = None
break
if has_optional or p.is_optional():
has_optional = True
parser_code.append(normalize_snippet("""
if (%s < %d) {{
goto skip_optional;
}}
""", indent=4) % (nargs, i + 1))
parser_code.append(normalize_snippet(parsearg, indent=4))
if parser_code is not None:
if has_optional:
parser_code.append("skip_optional:")
else:
if not new_or_init:
parser_code = [normalize_snippet("""
if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}",
{parse_arguments})) {{
goto exit;
}}
""", indent=4)]
else:
parser_code = [normalize_snippet("""
if (!PyArg_ParseTuple(args, "{format_units}:{name}",
{parse_arguments})) {{
goto exit;
}}
""", indent=4)]
parser_definition = parser_body(parser_prototype, *parser_code)
else:
has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG))
if vararg == NO_VARARG:
args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % (
min_pos,
max_pos,
min_kw_only
)
nargs = "nargs"
else:
args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % (
min_pos,
max_pos,
min_kw_only,
vararg
)
nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0"
if not new_or_init:
flags = "METH_FASTCALL|METH_KEYWORDS"
parser_prototype = parser_prototype_fastcall_keywords
argname_fmt = 'args[%d]'
declarations = declare_parser(f)
declarations += "\nPyObject *argsbuf[%s];" % len(converters)
if has_optional_kw:
declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only)
parser_code = [normalize_snippet("""
args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf);
if (!args) {{
goto exit;
}}
""" % args_declaration, indent=4)]
else:
# positional-or-keyword arguments
flags = "METH_VARARGS|METH_KEYWORDS"
parser_prototype = parser_prototype_keyword
argname_fmt = 'fastargs[%d]'
declarations = declare_parser(f)
declarations += "\nPyObject *argsbuf[%s];" % len(converters)
declarations += "\nPyObject * const *fastargs;"
declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);"
if has_optional_kw:
declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only)
parser_code = [normalize_snippet("""
fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf);
if (!fastargs) {{
goto exit;
}}
""" % args_declaration, indent=4)]
if requires_defining_class:
flags = 'METH_METHOD|' + flags
parser_prototype = parser_prototype_def_class
add_label = None
for i, p in enumerate(parameters):
if isinstance(p.converter, defining_class_converter):
raise ValueError("defining_class should be the first "
"parameter (after self)")
displayname = p.get_displayname(i+1)
parsearg = p.converter.parse_arg(argname_fmt % i, displayname)
if parsearg is None:
#print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr)
parser_code = None
break
if add_label and (i == pos_only or i == max_pos):
parser_code.append("%s:" % add_label)
add_label = None
if not p.is_optional():
parser_code.append(normalize_snippet(parsearg, indent=4))
elif i < pos_only:
add_label = 'skip_optional_posonly'
parser_code.append(normalize_snippet("""
if (nargs < %d) {{
goto %s;
}}
""" % (i + 1, add_label), indent=4))
if has_optional_kw:
parser_code.append(normalize_snippet("""
noptargs--;
""", indent=4))
parser_code.append(normalize_snippet(parsearg, indent=4))
else:
if i < max_pos:
label = 'skip_optional_pos'
first_opt = max(min_pos, pos_only)
else:
label = 'skip_optional_kwonly'
first_opt = max_pos + min_kw_only
if vararg != NO_VARARG:
first_opt += 1
if i == first_opt:
add_label = label
parser_code.append(normalize_snippet("""
if (!noptargs) {{
goto %s;
}}
""" % add_label, indent=4))
if i + 1 == len(parameters):
parser_code.append(normalize_snippet(parsearg, indent=4))
else:
add_label = label
parser_code.append(normalize_snippet("""
if (%s) {{
""" % (argname_fmt % i), indent=4))
parser_code.append(normalize_snippet(parsearg, indent=8))
parser_code.append(normalize_snippet("""
if (!--noptargs) {{
goto %s;
}}
}}
""" % add_label, indent=4))
if parser_code is not None:
if add_label:
parser_code.append("%s:" % add_label)
else:
declarations = declare_parser(f, hasformat=True)
if not new_or_init:
parser_code = [normalize_snippet("""
if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma}
{parse_arguments})) {{
goto exit;
}}
""", indent=4)]
else:
parser_code = [normalize_snippet("""
if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser,
{parse_arguments})) {{
goto exit;
}}
""", indent=4)]
parser_definition = parser_body(parser_prototype, *parser_code,
declarations=declarations)
if new_or_init:
methoddef_define = ''
if f.kind == METHOD_NEW:
parser_prototype = parser_prototype_keyword
else:
return_value_declaration = "int return_value = -1;"
parser_prototype = normalize_snippet("""
static int
{c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs)
""")
fields = list(parser_body_fields)
parses_positional = 'METH_NOARGS' not in flags
parses_keywords = 'METH_KEYWORDS' in flags
if parses_keywords:
assert parses_positional
if requires_defining_class:
raise ValueError("Slot methods cannot access their defining class.")
if not parses_keywords:
declarations = '{base_type_ptr}'
fields.insert(0, normalize_snippet("""
if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{
goto exit;
}}
""", indent=4))
if not parses_positional:
fields.insert(0, normalize_snippet("""
if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{
goto exit;
}}
""", indent=4))
parser_definition = parser_body(parser_prototype, *fields,
declarations=declarations)
if flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'):
methoddef_cast = "(PyCFunction)"
methoddef_cast_end = ""
else:
methoddef_cast = "_PyCFunction_CAST("
methoddef_cast_end = ")"
if f.methoddef_flags:
flags += '|' + f.methoddef_flags
methoddef_define = methoddef_define.replace('{methoddef_flags}', flags)
methoddef_define = methoddef_define.replace('{methoddef_cast}', methoddef_cast)
methoddef_define = methoddef_define.replace('{methoddef_cast_end}', methoddef_cast_end)
methoddef_ifndef = ''
conditional = self.cpp.condition()
if not conditional:
cpp_if = cpp_endif = ''
else:
cpp_if = "#if " + conditional
cpp_endif = "#endif /* " + conditional + " */"
if methoddef_define and f.full_name not in clinic.ifndef_symbols:
clinic.ifndef_symbols.add(f.full_name)
methoddef_ifndef = normalize_snippet("""
#ifndef {methoddef_name}
#define {methoddef_name}
#endif /* !defined({methoddef_name}) */
""")
# add ';' to the end of parser_prototype and impl_prototype
# (they mustn't be None, but they could be an empty string.)
assert parser_prototype is not None
if parser_prototype:
assert not parser_prototype.endswith(';')
parser_prototype += ';'
if impl_prototype is None:
impl_prototype = impl_definition
if impl_prototype:
impl_prototype += ";"
parser_definition = parser_definition.replace("{return_value_declaration}", return_value_declaration)
d = {
"docstring_prototype" : docstring_prototype,
"docstring_definition" : docstring_definition,
"impl_prototype" : impl_prototype,
"methoddef_define" : methoddef_define,
"parser_prototype" : parser_prototype,
"parser_definition" : parser_definition,
"impl_definition" : impl_definition,
"cpp_if" : cpp_if,
"cpp_endif" : cpp_endif,
"methoddef_ifndef" : methoddef_ifndef,
}
# make sure we didn't forget to assign something,
# and wrap each non-empty value in \n's
d2 = {}
for name, value in d.items():
assert value is not None, "got a None value for template " + repr(name)
if value:
value = '\n' + value + '\n'
d2[name] = value
return d2
@staticmethod
def group_to_variable_name(group):
adjective = "left_" if group < 0 else "right_"
return "group_" + adjective + str(abs(group))
def render_option_group_parsing(self, f, template_dict):
# positional only, grouped, optional arguments!
# can be optional on the left or right.
# here's an example:
#
# [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ]
#
# Here group D are required, and all other groups are optional.
# (Group D's "group" is actually None.)
# We can figure out which sets of arguments we have based on
# how many arguments are in the tuple.
#
# Note that you need to count up on both sides. For example,
# you could have groups C+D, or C+D+E, or C+D+E+F.
#
# What if the number of arguments leads us to an ambiguous result?
# Clinic prefers groups on the left. So in the above example,
# five arguments would map to B+C, not C+D.
add, output = text_accumulator()
parameters = list(f.parameters.values())
if isinstance(parameters[0].converter, self_converter):
del parameters[0]
group = None
left = []
right = []
required = []
last = unspecified
for p in parameters:
group_id = p.group
if group_id != last:
last = group_id
group = []
if group_id < 0:
left.append(group)
elif group_id == 0:
group = required
else:
right.append(group)
group.append(p)
count_min = sys.maxsize
count_max = -1
add("switch (PyTuple_GET_SIZE(args)) {\n")
for subset in permute_optional_groups(left, required, right):
count = len(subset)
count_min = min(count_min, count)
count_max = max(count_max, count)
if count == 0:
add(""" case 0:
break;
""")
continue
group_ids = {p.group for p in subset} # eliminate duplicates
d = {}
d['count'] = count
d['name'] = f.name
d['format_units'] = "".join(p.converter.format_unit for p in subset)
parse_arguments = []
for p in subset:
p.converter.parse_argument(parse_arguments)
d['parse_arguments'] = ", ".join(parse_arguments)
group_ids.discard(0)
lines = [self.group_to_variable_name(g) + " = 1;" for g in group_ids]
lines = "\n".join(lines)
s = """\
case {count}:
if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{
goto exit;
}}
{group_booleans}
break;
"""
s = linear_format(s, group_booleans=lines)
s = s.format_map(d)
add(s)
add(" default:\n")
s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n'
add(s.format(f.full_name, count_min, count_max))
add(' goto exit;\n')
add("}")
template_dict['option_group_parsing'] = format_escape(output())
def render_function(self, clinic, f):
if not f:
return ""
add, output = text_accumulator()
data = CRenderData()
assert f.parameters, "We should always have a 'self' at this point!"
parameters = f.render_parameters
converters = [p.converter for p in parameters]
templates = self.output_templates(f)
f_self = parameters[0]
selfless = parameters[1:]
assert isinstance(f_self.converter, self_converter), "No self parameter in " + repr(f.full_name) + "!"
last_group = 0
first_optional = len(selfless)
positional = selfless and selfless[-1].is_positional_only()
new_or_init = f.kind in (METHOD_NEW, METHOD_INIT)
has_option_groups = False
# offset i by -1 because first_optional needs to ignore self
for i, p in enumerate(parameters, -1):
c = p.converter
if (i != -1) and (p.default is not unspecified):
first_optional = min(first_optional, i)
if p.is_vararg():
data.cleanup.append(f"Py_XDECREF({c.parser_name});")
# insert group variable
group = p.group
if last_group != group:
last_group = group
if group:
group_name = self.group_to_variable_name(group)
data.impl_arguments.append(group_name)
data.declarations.append("int " + group_name + " = 0;")
data.impl_parameters.append("int " + group_name)
has_option_groups = True
c.render(p, data)
if has_option_groups and (not positional):
fail("You cannot use optional groups ('[' and ']')\nunless all parameters are positional-only ('/').")
# HACK
# when we're METH_O, but have a custom return converter,
# we use "impl_parameters" for the parsing function
# because that works better. but that means we must
# suppress actually declaring the impl's parameters
# as variables in the parsing function. but since it's
# METH_O, we have exactly one anyway, so we know exactly
# where it is.
if ("METH_O" in templates['methoddef_define'] and
'{impl_parameters}' in templates['parser_prototype']):
data.declarations.pop(0)
template_dict = {}
full_name = f.full_name
template_dict['full_name'] = full_name
if new_or_init:
name = f.cls.name
else:
name = f.name
template_dict['name'] = name
if f.c_basename:
c_basename = f.c_basename
else:
fields = full_name.split(".")
if fields[-1] == '__new__':
fields.pop()
c_basename = "_".join(fields)
template_dict['c_basename'] = c_basename
template_dict['methoddef_name'] = c_basename.upper() + "_METHODDEF"
template_dict['docstring'] = self.docstring_for_c_string(f)
template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = ''
for converter in converters:
converter.set_template_dict(template_dict)
f.return_converter.render(f, data)
template_dict['impl_return_type'] = f.return_converter.type
template_dict['declarations'] = format_escape("\n".join(data.declarations))
template_dict['initializers'] = "\n\n".join(data.initializers)
template_dict['modifications'] = '\n\n'.join(data.modifications)
template_dict['keywords_c'] = ' '.join('"' + k + '",'
for k in data.keywords)
keywords = [k for k in data.keywords if k]
template_dict['keywords_py'] = ' '.join('&_Py_ID(' + k + '),'
for k in keywords)
template_dict['format_units'] = ''.join(data.format_units)
template_dict['parse_arguments'] = ', '.join(data.parse_arguments)
if data.parse_arguments:
template_dict['parse_arguments_comma'] = ',';
else:
template_dict['parse_arguments_comma'] = '';
template_dict['impl_parameters'] = ", ".join(data.impl_parameters)
template_dict['impl_arguments'] = ", ".join(data.impl_arguments)
template_dict['return_conversion'] = format_escape("".join(data.return_conversion).rstrip())
template_dict['post_parsing'] = format_escape("".join(data.post_parsing).rstrip())
template_dict['cleanup'] = format_escape("".join(data.cleanup))
template_dict['return_value'] = data.return_value
# used by unpack tuple code generator
unpack_min = first_optional
unpack_max = len(selfless)
template_dict['unpack_min'] = str(unpack_min)
template_dict['unpack_max'] = str(unpack_max)
if has_option_groups:
self.render_option_group_parsing(f, template_dict)
# buffers, not destination
for name, destination in clinic.destination_buffers.items():
template = templates[name]
if has_option_groups:
template = linear_format(template,
option_group_parsing=template_dict['option_group_parsing'])
template = linear_format(template,
declarations=template_dict['declarations'],
return_conversion=template_dict['return_conversion'],
initializers=template_dict['initializers'],
modifications=template_dict['modifications'],
post_parsing=template_dict['post_parsing'],
cleanup=template_dict['cleanup'],
)
# Only generate the "exit:" label
# if we have any gotos
need_exit_label = "goto exit;" in template
template = linear_format(template,
exit_label="exit:" if need_exit_label else ''
)
s = template.format_map(template_dict)
# mild hack:
# reflow long impl declarations
if name in {"impl_prototype", "impl_definition"}:
s = wrap_declarations(s)
if clinic.line_prefix:
s = indent_all_lines(s, clinic.line_prefix)
if clinic.line_suffix:
s = suffix_all_lines(s, clinic.line_suffix)
destination.append(s)
return clinic.get_destination('block').dump()
@contextlib.contextmanager
def OverrideStdioWith(stdout):
saved_stdout = sys.stdout
sys.stdout = stdout
try:
yield
finally:
assert sys.stdout is stdout
sys.stdout = saved_stdout
def create_regex(before, after, word=True, whole_line=True):
"""Create an re object for matching marker lines."""
group_re = r"\w+" if word else ".+"
pattern = r'{}({}){}'
if whole_line:
pattern = '^' + pattern + '$'
pattern = pattern.format(re.escape(before), group_re, re.escape(after))
return re.compile(pattern)
class Block:
r"""
Represents a single block of text embedded in
another file. If dsl_name is None, the block represents
verbatim text, raw original text from the file, in
which case "input" will be the only non-false member.
If dsl_name is not None, the block represents a Clinic
block.
input is always str, with embedded \n characters.
input represents the original text from the file;
if it's a Clinic block, it is the original text with
the body_prefix and redundant leading whitespace removed.
dsl_name is either str or None. If str, it's the text
found on the start line of the block between the square
brackets.
signatures is either list or None. If it's a list,
it may only contain clinic.Module, clinic.Class, and
clinic.Function objects. At the moment it should
contain at most one of each.
output is either str or None. If str, it's the output
from this block, with embedded '\n' characters.
indent is either str or None. It's the leading whitespace
that was found on every line of input. (If body_prefix is
not empty, this is the indent *after* removing the
body_prefix.)
preindent is either str or None. It's the whitespace that
was found in front of every line of input *before* the
"body_prefix" (see the Language object). If body_prefix
is empty, preindent must always be empty too.
To illustrate indent and preindent: Assume that '_'
represents whitespace. If the block processed was in a
Python file, and looked like this:
____#/*[python]
____#__for a in range(20):
____#____print(a)
____#[python]*/
"preindent" would be "____" and "indent" would be "__".
"""
def __init__(self, input, dsl_name=None, signatures=None, output=None, indent='', preindent=''):
assert isinstance(input, str)
self.input = input
self.dsl_name = dsl_name
self.signatures = signatures or []
self.output = output
self.indent = indent
self.preindent = preindent
def __repr__(self):
dsl_name = self.dsl_name or "text"
def summarize(s):
s = repr(s)
if len(s) > 30:
return s[:26] + "..." + s[0]
return s
return "".join((
"<Block ", dsl_name, " input=", summarize(self.input), " output=", summarize(self.output), ">"))
class BlockParser:
"""
Block-oriented parser for Argument Clinic.
Iterator, yields Block objects.
"""
def __init__(self, input, language, *, verify=True):
"""
"input" should be a str object
with embedded \n characters.
"language" should be a Language object.
"""
language.validate()
self.input = collections.deque(reversed(input.splitlines(keepends=True)))
self.block_start_line_number = self.line_number = 0
self.language = language
before, _, after = language.start_line.partition('{dsl_name}')
assert _ == '{dsl_name}'
self.find_start_re = create_regex(before, after, whole_line=False)
self.start_re = create_regex(before, after)
self.verify = verify
self.last_checksum_re = None
self.last_dsl_name = None
self.dsl_name = None
self.first_block = True
def __iter__(self):
return self
def __next__(self):
while True:
if not self.input:
raise StopIteration
if self.dsl_name:
return_value = self.parse_clinic_block(self.dsl_name)
self.dsl_name = None
self.first_block = False
return return_value
block = self.parse_verbatim_block()
if self.first_block and not block.input:
continue
self.first_block = False
return block
def is_start_line(self, line):
match = self.start_re.match(line.lstrip())
return match.group(1) if match else None
def _line(self, lookahead=False):
self.line_number += 1
line = self.input.pop()
if not lookahead:
self.language.parse_line(line)
return line
def parse_verbatim_block(self):
add, output = text_accumulator()
self.block_start_line_number = self.line_number
while self.input:
line = self._line()
dsl_name = self.is_start_line(line)
if dsl_name:
self.dsl_name = dsl_name
break
add(line)
return Block(output())
def parse_clinic_block(self, dsl_name):
input_add, input_output = text_accumulator()
self.block_start_line_number = self.line_number + 1
stop_line = self.language.stop_line.format(dsl_name=dsl_name)
body_prefix = self.language.body_prefix.format(dsl_name=dsl_name)
def is_stop_line(line):
# make sure to recognize stop line even if it
# doesn't end with EOL (it could be the very end of the file)
if line.startswith(stop_line):
remainder = line.removeprefix(stop_line)
if remainder and not remainder.isspace():
fail(f"Garbage after stop line: {remainder!r}")
return True
else:
# gh-92256: don't allow incorrectly formatted stop lines
if line.lstrip().startswith(stop_line):
fail(f"Whitespace is not allowed before the stop line: {line!r}")
return False
# consume body of program
while self.input:
line = self._line()
if is_stop_line(line) or self.is_start_line(line):
break
if body_prefix:
line = line.lstrip()
assert line.startswith(body_prefix)
line = line.removeprefix(body_prefix)
input_add(line)
# consume output and checksum line, if present.
if self.last_dsl_name == dsl_name:
checksum_re = self.last_checksum_re
else:
before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}')
assert _ == '{arguments}'
checksum_re = create_regex(before, after, word=False)
self.last_dsl_name = dsl_name
self.last_checksum_re = checksum_re
# scan forward for checksum line
output_add, output_output = text_accumulator()
arguments = None
while self.input:
line = self._line(lookahead=True)
match = checksum_re.match(line.lstrip())
arguments = match.group(1) if match else None
if arguments:
break
output_add(line)
if self.is_start_line(line):
break
output = output_output()
if arguments:
d = {}
for field in shlex.split(arguments):
name, equals, value = field.partition('=')
if not equals:
fail("Mangled Argument Clinic marker line:", repr(line))
d[name.strip()] = value.strip()
if self.verify:
if 'input' in d:
checksum = d['output']
else:
checksum = d['checksum']
computed = compute_checksum(output, len(checksum))
if checksum != computed:
fail("Checksum mismatch!\nExpected: {}\nComputed: {}\n"
"Suggested fix: remove all generated code including "
"the end marker,\n"
"or use the '-f' option."
.format(checksum, computed))
else:
# put back output
output_lines = output.splitlines(keepends=True)
self.line_number -= len(output_lines)
self.input.extend(reversed(output_lines))
output = None
return Block(input_output(), dsl_name, output=output)
class BlockPrinter:
def __init__(self, language, f=None):
self.language = language
self.f = f or io.StringIO()
def print_block(self, block, *, core_includes=False):
input = block.input
output = block.output
dsl_name = block.dsl_name
write = self.f.write
assert not ((dsl_name is None) ^ (output is None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name)
if not dsl_name:
write(input)
return
write(self.language.start_line.format(dsl_name=dsl_name))
write("\n")
body_prefix = self.language.body_prefix.format(dsl_name=dsl_name)
if not body_prefix:
write(input)
else:
for line in input.split('\n'):
write(body_prefix)
write(line)
write("\n")
write(self.language.stop_line.format(dsl_name=dsl_name))
write("\n")
output = ''
if core_includes:
output += textwrap.dedent("""
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
# include "pycore_gc.h" // PyGC_Head
# include "pycore_runtime.h" // _Py_ID()
#endif
""")
input = ''.join(block.input)
output += ''.join(block.output)
if output:
if not output.endswith('\n'):
output += '\n'
write(output)
arguments = "output={output} input={input}".format(
output=compute_checksum(output, 16),
input=compute_checksum(input, 16)
)
write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments))
write("\n")
def write(self, text):
self.f.write(text)
class BufferSeries:
"""
Behaves like a "defaultlist".
When you ask for an index that doesn't exist yet,
the object grows the list until that item exists.
So o[n] will always work.
Supports negative indices for actual items.
e.g. o[-1] is an element immediately preceding o[0].
"""
def __init__(self):
self._start = 0
self._array = []
self._constructor = _text_accumulator
def __getitem__(self, i):
i -= self._start
if i < 0:
self._start += i
prefix = [self._constructor() for x in range(-i)]
self._array = prefix + self._array
i = 0
while i >= len(self._array):
self._array.append(self._constructor())
return self._array[i]
def clear(self):
for ta in self._array:
ta._text.clear()
def dump(self):
texts = [ta.output() for ta in self._array]
return "".join(texts)
class Destination:
def __init__(self, name, type, clinic, *args):
self.name = name
self.type = type
self.clinic = clinic
valid_types = ('buffer', 'file', 'suppress')
if type not in valid_types:
fail("Invalid destination type " + repr(type) + " for " + name + " , must be " + ', '.join(valid_types))
extra_arguments = 1 if type == "file" else 0
if len(args) < extra_arguments:
fail("Not enough arguments for destination " + name + " new " + type)
if len(args) > extra_arguments:
fail("Too many arguments for destination " + name + " new " + type)
if type =='file':
d = {}
filename = clinic.filename
d['path'] = filename
dirname, basename = os.path.split(filename)
if not dirname:
dirname = '.'
d['dirname'] = dirname
d['basename'] = basename
d['basename_root'], d['basename_extension'] = os.path.splitext(filename)
self.filename = args[0].format_map(d)
self.buffers = BufferSeries()
def __repr__(self):
if self.type == 'file':
file_repr = " " + repr(self.filename)
else:
file_repr = ''
return "".join(("<Destination ", self.name, " ", self.type, file_repr, ">"))
def clear(self):
if self.type != 'buffer':
fail("Can't clear destination" + self.name + " , it's not of type buffer")
self.buffers.clear()
def dump(self):
return self.buffers.dump()
# maps strings to Language objects.
# "languages" maps the name of the language ("C", "Python").
# "extensions" maps the file extension ("c", "py").
LangDict = dict[str, Callable[[str], Language]]
languages = { 'C': CLanguage, 'Python': PythonLanguage }
extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() }
extensions['py'] = PythonLanguage
def file_changed(filename: str, new_contents: str) -> bool:
"""Return true if file contents changed (meaning we must update it)"""
try:
with open(filename, encoding="utf-8") as fp:
old_contents = fp.read()
return old_contents != new_contents
except FileNotFoundError:
return True
def write_file(filename: str, new_contents: str):
# Atomic write using a temporary file and os.replace()
filename_new = f"{filename}.new"
with open(filename_new, "w", encoding="utf-8") as fp:
fp.write(new_contents)
try:
os.replace(filename_new, filename)
except:
os.unlink(filename_new)
raise
ClassDict = dict[str, "Class"]
DestinationDict = dict[str, Destination]
ModuleDict = dict[str, "Module"]
ParserDict = dict[str, "DSLParser"]
clinic = None
class Clinic:
presets_text = """
preset block
everything block
methoddef_ifndef buffer 1
docstring_prototype suppress
parser_prototype suppress
cpp_if suppress
cpp_endif suppress
preset original
everything block
methoddef_ifndef buffer 1
docstring_prototype suppress
parser_prototype suppress
cpp_if suppress
cpp_endif suppress
preset file
everything file
methoddef_ifndef file 1
docstring_prototype suppress
parser_prototype suppress
impl_definition block
preset buffer
everything buffer
methoddef_ifndef buffer 1
impl_definition block
docstring_prototype suppress
impl_prototype suppress
parser_prototype suppress
preset partial-buffer
everything buffer
methoddef_ifndef buffer 1
docstring_prototype block
impl_prototype suppress
methoddef_define block
parser_prototype block
impl_definition block
"""
def __init__(
self,
language: CLanguage,
printer: BlockPrinter | None = None,
*,
verify: bool = True,
filename: str | None = None
) -> None:
# maps strings to Parser objects.
# (instantiated from the "parsers" global.)
self.parsers: ParserDict = {}
self.language: CLanguage = language
if printer:
fail("Custom printers are broken right now")
self.printer = printer or BlockPrinter(language)
self.verify = verify
self.filename = filename
self.modules: ModuleDict = {}
self.classes: ClassDict = {}
self.functions: list[Function] = []
self.line_prefix = self.line_suffix = ''
self.destinations: DestinationDict = {}
self.add_destination("block", "buffer")
self.add_destination("suppress", "suppress")
self.add_destination("buffer", "buffer")
if filename:
self.add_destination("file", "file", "{dirname}/clinic/{basename}.h")
d = self.get_destination_buffer
self.destination_buffers = {
'cpp_if': d('file'),
'docstring_prototype': d('suppress'),
'docstring_definition': d('file'),
'methoddef_define': d('file'),
'impl_prototype': d('file'),
'parser_prototype': d('suppress'),
'parser_definition': d('file'),
'cpp_endif': d('file'),
'methoddef_ifndef': d('file', 1),
'impl_definition': d('block'),
}
DestBufferType = dict[str, Callable[..., Any]]
DestBufferList = list[DestBufferType]
self.destination_buffers_stack: DestBufferList = []
self.ifndef_symbols: set[str] = set()
self.presets: dict[str, dict[Any, Any]] = {}
preset = None
for line in self.presets_text.strip().split('\n'):
line = line.strip()
if not line:
continue
name, value, *options = line.split()
if name == 'preset':
self.presets[value] = preset = {}
continue
if len(options):
index = int(options[0])
else:
index = 0
buffer = self.get_destination_buffer(value, index)
if name == 'everything':
for name in self.destination_buffers:
preset[name] = buffer
continue
assert name in self.destination_buffers
preset[name] = buffer
global clinic
clinic = self
def add_destination(
self,
name: str,
type: str,
*args
) -> None:
if name in self.destinations:
fail("Destination already exists: " + repr(name))
self.destinations[name] = Destination(name, type, self, *args)
def get_destination(self, name: str) -> Destination:
d = self.destinations.get(name)
if not d:
fail("Destination does not exist: " + repr(name))
return d
def get_destination_buffer(
self,
name: str,
item: int = 0
):
d = self.get_destination(name)
return d.buffers[item]
def parse(self, input):
printer = self.printer
self.block_parser = BlockParser(input, self.language, verify=self.verify)
for block in self.block_parser:
dsl_name = block.dsl_name
if dsl_name:
if dsl_name not in self.parsers:
assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block."
self.parsers[dsl_name] = parsers[dsl_name](self)
parser = self.parsers[dsl_name]
try:
parser.parse(block)
except Exception:
fail('Exception raised during parsing:\n' +
traceback.format_exc().rstrip())
printer.print_block(block)
clinic_out = []
# these are destinations not buffers
for name, destination in self.destinations.items():
if destination.type == 'suppress':
continue
output = destination.dump()
if output:
block = Block("", dsl_name="clinic", output=output)
if destination.type == 'buffer':
block.input = "dump " + name + "\n"
warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.")
printer.write("\n")
printer.print_block(block)
continue
if destination.type == 'file':
try:
dirname = os.path.dirname(destination.filename)
try:
os.makedirs(dirname)
except FileExistsError:
if not os.path.isdir(dirname):
fail("Can't write to destination {}, "
"can't make directory {}!".format(
destination.filename, dirname))
if self.verify:
with open(destination.filename) as f:
parser_2 = BlockParser(f.read(), language=self.language)
blocks = list(parser_2)
if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'):
fail("Modified destination file " + repr(destination.filename) + ", not overwriting!")
except FileNotFoundError:
pass
block.input = 'preserve\n'
printer_2 = BlockPrinter(self.language)
printer_2.print_block(block, core_includes=True)
pair = destination.filename, printer_2.f.getvalue()
clinic_out.append(pair)
continue
return printer.f.getvalue(), clinic_out
def _module_and_class(self, fields):
"""
fields should be an iterable of field names.
returns a tuple of (module, class).
the module object could actually be self (a clinic object).
this function is only ever used to find the parent of where
a new class/module should go.
"""
in_classes = False
parent = module = self
cls = None
so_far = []
for field in fields:
so_far.append(field)
if not in_classes:
child = parent.modules.get(field)
if child:
parent = module = child
continue
in_classes = True
if not hasattr(parent, 'classes'):
return module, cls
child = parent.classes.get(field)
if not child:
fail('Parent class or module ' + '.'.join(so_far) + " does not exist.")
cls = parent = child
return module, cls
def parse_file(
filename: str,
*,
verify: bool = True,
output: str | None = None
) -> None:
if not output:
output = filename
extension = os.path.splitext(filename)[1][1:]
if not extension:
fail("Can't extract file type for file " + repr(filename))
try:
language = extensions[extension](filename)
except KeyError:
fail("Can't identify file type for file " + repr(filename))
with open(filename, encoding="utf-8") as f:
raw = f.read()
# exit quickly if there are no clinic markers in the file
find_start_re = BlockParser("", language).find_start_re
if not find_start_re.search(raw):
return
assert isinstance(language, CLanguage)
clinic = Clinic(language, verify=verify, filename=filename)
src_out, clinic_out = clinic.parse(raw)
changes = [(fn, data) for fn, data in clinic_out if file_changed(fn, data)]
if changes:
# Always (re)write the source file.
write_file(output, src_out)
for fn, data in clinic_out:
write_file(fn, data)
def compute_checksum(
input: str | None,
length: int | None = None
) -> str:
input = input or ''
s = hashlib.sha1(input.encode('utf-8')).hexdigest()
if length:
s = s[:length]
return s
class PythonParser:
def __init__(self, clinic: Clinic) -> None:
pass
def parse(self, block: Block) -> None:
s = io.StringIO()
with OverrideStdioWith(s):
exec(block.input)
block.output = s.getvalue()
class Module:
def __init__(
self,
name: str,
module = None
) -> None:
self.name = name
self.module = self.parent = module
self.modules: ModuleDict = {}
self.classes: ClassDict = {}
self.functions: list[Function] = []
def __repr__(self) -> str:
return "<clinic.Module " + repr(self.name) + " at " + str(id(self)) + ">"
class Class:
def __init__(
self,
name: str,
module: Module | None = None,
cls = None,
typedef: str | None = None,
type_object: str | None = None
) -> None:
self.name = name
self.module = module
self.cls = cls
self.typedef = typedef
self.type_object = type_object
self.parent = cls or module
self.classes: ClassDict = {}
self.functions: list[Function] = []
def __repr__(self) -> str:
return "<clinic.Class " + repr(self.name) + " at " + str(id(self)) + ">"
unsupported_special_methods: set[str] = set("""
__abs__
__add__
__and__
__call__
__delitem__
__divmod__
__eq__
__float__
__floordiv__
__ge__
__getattr__
__getattribute__
__getitem__
__gt__
__hash__
__iadd__
__iand__
__ifloordiv__
__ilshift__
__imatmul__
__imod__
__imul__
__index__
__int__
__invert__
__ior__
__ipow__
__irshift__
__isub__
__iter__
__itruediv__
__ixor__
__le__
__len__
__lshift__
__lt__
__matmul__
__mod__
__mul__
__neg__
__next__
__or__
__pos__
__pow__
__radd__
__rand__
__rdivmod__
__repr__
__rfloordiv__
__rlshift__
__rmatmul__
__rmod__
__rmul__
__ror__
__rpow__
__rrshift__
__rshift__
__rsub__
__rtruediv__
__rxor__
__setattr__
__setitem__
__str__
__sub__
__truediv__
__xor__
""".strip().split())
INVALID, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW = """
INVALID, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW
""".replace(",", "").strip().split()
ParamDict = dict[str, "Parameter"]
ReturnConverterType = Callable[..., "CReturnConverter"]
class Function:
"""
Mutable duck type for inspect.Function.
docstring - a str containing
* embedded line breaks
* text outdented to the left margin
* no trailing whitespace.
It will always be true that
(not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring))
"""
def __init__(
self,
parameters: ParamDict | None = None,
*,
name: str,
module: Module,
cls: Class | None = None,
c_basename: str | None = None,
full_name: str | None = None,
return_converter: ReturnConverterType,
return_annotation = inspect.Signature.empty,
docstring: str | None = None,
kind: str = CALLABLE,
coexist: bool = False,
docstring_only: bool = False
) -> None:
self.parameters = parameters or {}
self.return_annotation = return_annotation
self.name = name
self.full_name = full_name
self.module = module
self.cls = cls
self.parent = cls or module
self.c_basename = c_basename
self.return_converter = return_converter
self.docstring = docstring or ''
self.kind = kind
self.coexist = coexist
self.self_converter = None
# docstring_only means "don't generate a machine-readable
# signature, just a normal docstring". it's True for
# functions with optional groups because we can't represent
# those accurately with inspect.Signature in 3.4.
self.docstring_only = docstring_only
self.rendered_parameters = None
__render_parameters__ = None
@property
def render_parameters(self):
if not self.__render_parameters__:
self.__render_parameters__ = l = []
for p in self.parameters.values():
p = p.copy()
p.converter.pre_render()
l.append(p)
return self.__render_parameters__
@property
def methoddef_flags(self) -> str | None:
if self.kind in (METHOD_INIT, METHOD_NEW):
return None
flags = []
if self.kind == CLASS_METHOD:
flags.append('METH_CLASS')
elif self.kind == STATIC_METHOD:
flags.append('METH_STATIC')
else:
assert self.kind == CALLABLE, "unknown kind: " + repr(self.kind)
if self.coexist:
flags.append('METH_COEXIST')
return '|'.join(flags)
def __repr__(self) -> str:
return '<clinic.Function ' + self.name + '>'
def copy(self, **overrides) -> "Function":
kwargs = {
'name': self.name, 'module': self.module, 'parameters': self.parameters,
'cls': self.cls, 'c_basename': self.c_basename,
'full_name': self.full_name,
'return_converter': self.return_converter, 'return_annotation': self.return_annotation,
'docstring': self.docstring, 'kind': self.kind, 'coexist': self.coexist,
'docstring_only': self.docstring_only,
}
kwargs.update(overrides)
f = Function(**kwargs)
f.parameters = {
name: value.copy(function=f)
for name, value in f.parameters.items()
}
return f
class Parameter:
"""
Mutable duck type of inspect.Parameter.
"""
def __init__(
self,
name: str,
kind: str,
*,
default = inspect.Parameter.empty,
function: Function,
converter: "CConverter",
annotation = inspect.Parameter.empty,
docstring: str | None = None,
group: int = 0
) -> None:
self.name = name
self.kind = kind
self.default = default
self.function = function
self.converter = converter
self.annotation = annotation
self.docstring = docstring or ''
self.group = group
def __repr__(self) -> str:
return '<clinic.Parameter ' + self.name + '>'
def is_keyword_only(self) -> bool:
return self.kind == inspect.Parameter.KEYWORD_ONLY
def is_positional_only(self) -> bool:
return self.kind == inspect.Parameter.POSITIONAL_ONLY
def is_vararg(self) -> bool:
return self.kind == inspect.Parameter.VAR_POSITIONAL
def is_optional(self) -> bool:
return not self.is_vararg() and (self.default is not unspecified)
def copy(self, **overrides) -> "Parameter":
kwargs = {
'name': self.name, 'kind': self.kind, 'default':self.default,
'function': self.function, 'converter': self.converter, 'annotation': self.annotation,
'docstring': self.docstring, 'group': self.group,
}
kwargs.update(overrides)
if 'converter' not in overrides:
converter = copy.copy(self.converter)
converter.function = kwargs['function']
kwargs['converter'] = converter
return Parameter(**kwargs)
def get_displayname(self, i: int) -> str:
if i == 0:
return '"argument"'
if not self.is_positional_only():
return f'"argument {self.name!r}"'
else:
return f'"argument {i}"'
class LandMine:
# try to access any
def __init__(self, message: str) -> None:
self.__message__ = message
def __repr__(self) -> str:
return '<LandMine ' + repr(self.__message__) + ">"
def __getattribute__(self, name: str):
if name in ('__repr__', '__message__'):
return super().__getattribute__(name)
# raise RuntimeError(repr(name))
fail("Stepped on a land mine, trying to access attribute " + repr(name) + ":\n" + self.__message__)
def add_c_converter(f, name=None):
if not name:
name = f.__name__
if not name.endswith('_converter'):
return f
name = name.removesuffix('_converter')
converters[name] = f
return f
def add_default_legacy_c_converter(cls):
# automatically add converter for default format unit
# (but without stomping on the existing one if it's already
# set, in case you subclass)
if ((cls.format_unit not in ('O&', '')) and
(cls.format_unit not in legacy_converters)):
legacy_converters[cls.format_unit] = cls
return cls
def add_legacy_c_converter(format_unit, **kwargs):
"""
Adds a legacy converter.
"""
def closure(f):
if not kwargs:
added_f = f
else:
added_f = functools.partial(f, **kwargs)
if format_unit:
legacy_converters[format_unit] = added_f
return f
return closure
class CConverterAutoRegister(type):
def __init__(cls, name, bases, classdict):
add_c_converter(cls)
add_default_legacy_c_converter(cls)
class CConverter(metaclass=CConverterAutoRegister):
"""
For the init function, self, name, function, and default
must be keyword-or-positional parameters. All other
parameters must be keyword-only.
"""
# The C name to use for this variable.
name: str | None = None
# The Python name to use for this variable.
py_name: str | None = None
# The C type to use for this variable.
# 'type' should be a Python string specifying the type, e.g. "int".
# If this is a pointer type, the type string should end with ' *'.
type: str | None = None
# The Python default value for this parameter, as a Python value.
# Or the magic value "unspecified" if there is no default.
# Or the magic value "unknown" if this value is a cannot be evaluated
# at Argument-Clinic-preprocessing time (but is presumed to be valid
# at runtime).
default: object = unspecified
# If not None, default must be isinstance() of this type.
# (You can also specify a tuple of types.)
default_type: bltns.type[Any] | tuple[bltns.type[Any], ...] | None = None
# "default" converted into a C value, as a string.
# Or None if there is no default.
c_default: str | None = None
# "default" converted into a Python value, as a string.
# Or None if there is no default.
py_default: str | None = None
# The default value used to initialize the C variable when
# there is no default, but not specifying a default may
# result in an "uninitialized variable" warning. This can
# easily happen when using option groups--although
# properly-written code won't actually use the variable,
# the variable does get passed in to the _impl. (Ah, if
# only dataflow analysis could inline the static function!)
#
# This value is specified as a string.
# Every non-abstract subclass should supply a valid value.
c_ignored_default: str = 'NULL'
# If true, wrap with Py_UNUSED.
unused = False
# The C converter *function* to be used, if any.
# (If this is not None, format_unit must be 'O&'.)
converter: str | None = None
# Should Argument Clinic add a '&' before the name of
# the variable when passing it into the _impl function?
impl_by_reference = False
# Should Argument Clinic add a '&' before the name of
# the variable when passing it into PyArg_ParseTuple (AndKeywords)?
parse_by_reference = True
#############################################################
#############################################################
## You shouldn't need to read anything below this point to ##
## write your own converter functions. ##
#############################################################
#############################################################
# The "format unit" to specify for this variable when
# parsing arguments using PyArg_ParseTuple (AndKeywords).
# Custom converters should always use the default value of 'O&'.
format_unit = 'O&'
# What encoding do we want for this variable? Only used
# by format units starting with 'e'.
encoding: str | None = None
# Should this object be required to be a subclass of a specific type?
# If not None, should be a string representing a pointer to a
# PyTypeObject (e.g. "&PyUnicode_Type").
# Only used by the 'O!' format unit (and the "object" converter).
subclass_of = None
# Do we want an adjacent '_length' variable for this variable?
# Only used by format units ending with '#'.
length = False
# Should we show this parameter in the generated
# __text_signature__? This is *almost* always True.
# (It's only False for __new__, __init__, and METH_STATIC functions.)
show_in_signature = True
# Overrides the name used in a text signature.
# The name used for a "self" parameter must be one of
# self, type, or module; however users can set their own.
# This lets the self_converter overrule the user-settable
# name, *just* for the text signature.
# Only set by self_converter.
signature_name = None
# keep in sync with self_converter.__init__!
def __init__(self,
# Positional args:
name: str,
py_name: str,
function,
default: object = unspecified,
*, # Keyword only args:
c_default: str | None = None,
py_default: str | None = None,
annotation: str | Literal[Sentinels.unspecified] = unspecified,
unused: bool = False,
**kwargs
):
self.name = ensure_legal_c_identifier(name)
self.py_name = py_name
self.unused = unused
if default is not unspecified:
if (self.default_type
and default is not unknown
and not isinstance(default, self.default_type)
):
if isinstance(self.default_type, type):
types_str = self.default_type.__name__
else:
names = [cls.__name__ for cls in self.default_type]
types_str = ', '.join(names)
fail("{}: default value {!r} for field {} is not of type {}".format(
self.__class__.__name__, default, name, types_str))
self.default = default
if c_default:
self.c_default = c_default
if py_default:
self.py_default = py_default
if annotation is not unspecified:
fail("The 'annotation' parameter is not currently permitted.")
# this is deliberate, to prevent you from caching information
# about the function in the init.
# (that breaks if we get cloned.)
# so after this change we will noisily fail.
self.function = LandMine("Don't access members of self.function inside converter_init!")
self.converter_init(**kwargs)
self.function = function
def converter_init(self):
pass
def is_optional(self) -> bool:
return (self.default is not unspecified)
def _render_self(self, parameter: str, data: CRenderData) -> None:
self.parameter = parameter
name = self.parser_name
# impl_arguments
s = ("&" if self.impl_by_reference else "") + name
data.impl_arguments.append(s)
if self.length:
data.impl_arguments.append(self.length_name())
# impl_parameters
data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference))
if self.length:
data.impl_parameters.append("Py_ssize_t " + self.length_name())
def _render_non_self(self, parameter, data):
self.parameter = parameter
name = self.name
# declarations
d = self.declaration(in_parser=True)
data.declarations.append(d)
# initializers
initializers = self.initialize()
if initializers:
data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip())
# modifications
modifications = self.modify()
if modifications:
data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip())
# keywords
if parameter.is_vararg():
pass
elif parameter.is_positional_only():
data.keywords.append('')
else:
data.keywords.append(parameter.name)
# format_units
if self.is_optional() and '|' not in data.format_units:
data.format_units.append('|')
if parameter.is_keyword_only() and '$' not in data.format_units:
data.format_units.append('$')
data.format_units.append(self.format_unit)
# parse_arguments
self.parse_argument(data.parse_arguments)
# post_parsing
if post_parsing := self.post_parsing():
data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n')
# cleanup
cleanup = self.cleanup()
if cleanup:
data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n")
def render(self, parameter: str, data: CRenderData) -> None:
"""
parameter is a clinic.Parameter instance.
data is a CRenderData instance.
"""
self._render_self(parameter, data)
self._render_non_self(parameter, data)
def length_name(self):
"""Computes the name of the associated "length" variable."""
if not self.length:
return None
return self.parser_name + "_length"
# Why is this one broken out separately?
# For "positional-only" function parsing,
# which generates a bunch of PyArg_ParseTuple calls.
def parse_argument(self, list):
assert not (self.converter and self.encoding)
if self.format_unit == 'O&':
assert self.converter
list.append(self.converter)
if self.encoding:
list.append(c_repr(self.encoding))
elif self.subclass_of:
list.append(self.subclass_of)
s = ("&" if self.parse_by_reference else "") + self.name
list.append(s)
if self.length:
list.append("&" + self.length_name())
#
# All the functions after here are intended as extension points.
#
def simple_declaration(self, by_reference=False, *, in_parser=False):
"""
Computes the basic declaration of the variable.
Used in computing the prototype declaration and the
variable declaration.
"""
prototype = [self.type]
if by_reference or not self.type.endswith('*'):
prototype.append(" ")
if by_reference:
prototype.append('*')
if in_parser:
name = self.parser_name
else:
name = self.name
if self.unused:
name = f"Py_UNUSED({name})"
prototype.append(name)
return "".join(prototype)
def declaration(self, *, in_parser=False):
"""
The C statement to declare this variable.
"""
declaration = [self.simple_declaration(in_parser=True)]
default = self.c_default
if not default and self.parameter.group:
default = self.c_ignored_default
if default:
declaration.append(" = ")
declaration.append(default)
declaration.append(";")
if self.length:
declaration.append('\nPy_ssize_t ')
declaration.append(self.length_name())
declaration.append(';')
return "".join(declaration)
def initialize(self) -> str:
"""
The C statements required to set up this variable before parsing.
Returns a string containing this code indented at column 0.
If no initialization is necessary, returns an empty string.
"""
return ""
def modify(self) -> str:
"""
The C statements required to modify this variable after parsing.
Returns a string containing this code indented at column 0.
If no modification is necessary, returns an empty string.
"""
return ""
def post_parsing(self) -> str:
"""
The C statements required to do some operations after the end of parsing but before cleaning up.
Return a string containing this code indented at column 0.
If no operation is necessary, return an empty string.
"""
return ""
def cleanup(self) -> str:
"""
The C statements required to clean up after this variable.
Returns a string containing this code indented at column 0.
If no cleanup is necessary, returns an empty string.
"""
return ""
def pre_render(self):
"""
A second initialization function, like converter_init,
called just before rendering.
You are permitted to examine self.function here.
"""
pass
def parse_arg(self, argname, displayname):
if self.format_unit == 'O&':
return """
if (!{converter}({argname}, &{paramname})) {{{{
goto exit;
}}}}
""".format(argname=argname, paramname=self.parser_name,
converter=self.converter)
if self.format_unit == 'O!':
cast = '(%s)' % self.type if self.type != 'PyObject *' else ''
if self.subclass_of in type_checks:
typecheck, typename = type_checks[self.subclass_of]
return """
if (!{typecheck}({argname})) {{{{
_PyArg_BadArgument("{{name}}", {displayname}, "{typename}", {argname});
goto exit;
}}}}
{paramname} = {cast}{argname};
""".format(argname=argname, paramname=self.parser_name,
displayname=displayname, typecheck=typecheck,
typename=typename, cast=cast)
return """
if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{
_PyArg_BadArgument("{{name}}", {displayname}, ({subclass_of})->tp_name, {argname});
goto exit;
}}}}
{paramname} = {cast}{argname};
""".format(argname=argname, paramname=self.parser_name,
subclass_of=self.subclass_of, cast=cast,
displayname=displayname)
if self.format_unit == 'O':
cast = '(%s)' % self.type if self.type != 'PyObject *' else ''
return """
{paramname} = {cast}{argname};
""".format(argname=argname, paramname=self.parser_name, cast=cast)
return None
def set_template_dict(self, template_dict: TemplateDict) -> None:
pass
@property
def parser_name(self):
if self.name in CLINIC_PREFIXED_ARGS: # bpo-39741
return CLINIC_PREFIX + self.name
else:
return self.name
type_checks = {
'&PyLong_Type': ('PyLong_Check', 'int'),
'&PyTuple_Type': ('PyTuple_Check', 'tuple'),
'&PyList_Type': ('PyList_Check', 'list'),
'&PySet_Type': ('PySet_Check', 'set'),
'&PyFrozenSet_Type': ('PyFrozenSet_Check', 'frozenset'),
'&PyDict_Type': ('PyDict_Check', 'dict'),
'&PyUnicode_Type': ('PyUnicode_Check', 'str'),
'&PyBytes_Type': ('PyBytes_Check', 'bytes'),
'&PyByteArray_Type': ('PyByteArray_Check', 'bytearray'),
}
ConverterType = Callable[..., CConverter]
ConverterDict = dict[str, ConverterType]
# maps strings to callables.
# these callables must be of the form:
# def foo(name, default, *, ...)
# The callable may have any number of keyword-only parameters.
# The callable must return a CConverter object.
# The callable should not call builtins.print.
converters: ConverterDict = {}
# maps strings to callables.
# these callables follow the same rules as those for "converters" above.
# note however that they will never be called with keyword-only parameters.
legacy_converters: ConverterDict = {}
# maps strings to callables.
# these callables must be of the form:
# def foo(*, ...)
# The callable may have any number of keyword-only parameters.
# The callable must return a CReturnConverter object.
# The callable should not call builtins.print.
ReturnConverterDict = dict[str, ReturnConverterType]
return_converters: ReturnConverterDict = {}
TypeSet = set[bltns.type[Any]]
class bool_converter(CConverter):
type = 'int'
default_type = bool
format_unit = 'p'
c_ignored_default = '0'
def converter_init(self, *, accept: TypeSet = {object}) -> None:
if accept == {int}:
self.format_unit = 'i'
elif accept != {object}:
fail("bool_converter: illegal 'accept' argument " + repr(accept))
if self.default is not unspecified:
self