| #!/usr/bin/env python3 |
| # |
| # Argument Clinic |
| # Copyright 2012-2013 by Larry Hastings. |
| # Licensed to the PSF under a contributor agreement. |
| # |
| |
| import abc |
| import ast |
| import builtins as bltns |
| import collections |
| import contextlib |
| import copy |
| import cpp |
| import enum |
| import functools |
| import hashlib |
| import inspect |
| import io |
| import itertools |
| import os |
| import pprint |
| import re |
| import shlex |
| import string |
| import sys |
| import textwrap |
| import traceback |
| |
| from collections.abc import Callable |
| from types import FunctionType, NoneType |
| from typing import Any, Final, NamedTuple, NoReturn, Literal, overload |
| |
| # TODO: |
| # |
| # soon: |
| # |
| # * allow mixing any two of {positional-only, positional-or-keyword, |
| # keyword-only} |
| # * dict constructor uses positional-only and keyword-only |
| # * max and min use positional only with an optional group |
| # and keyword-only |
| # |
| |
| version = '1' |
| |
| NO_VARARG = "PY_SSIZE_T_MAX" |
| CLINIC_PREFIX = "__clinic_" |
| CLINIC_PREFIXED_ARGS = { |
| "_keywords", |
| "_parser", |
| "args", |
| "argsbuf", |
| "fastargs", |
| "kwargs", |
| "kwnames", |
| "nargs", |
| "noptargs", |
| "return_value", |
| } |
| |
| |
| class Sentinels(enum.Enum): |
| unspecified = "unspecified" |
| unknown = "unknown" |
| |
| def __repr__(self) -> str: |
| return f"<{self.value.capitalize()}>" |
| |
| |
| unspecified: Final = Sentinels.unspecified |
| unknown: Final = Sentinels.unknown |
| |
| |
| # This one needs to be a distinct class, unlike the other two |
| class Null: |
| def __repr__(self) -> str: |
| return '<Null>' |
| |
| |
| NULL = Null() |
| |
| sig_end_marker = '--' |
| |
| Appender = Callable[[str], None] |
| Outputter = Callable[[], str] |
| TemplateDict = dict[str, str] |
| |
| class _TextAccumulator(NamedTuple): |
| text: list[str] |
| append: Appender |
| output: Outputter |
| |
| def _text_accumulator() -> _TextAccumulator: |
| text: list[str] = [] |
| def output(): |
| s = ''.join(text) |
| text.clear() |
| return s |
| return _TextAccumulator(text, text.append, output) |
| |
| |
| class TextAccumulator(NamedTuple): |
| append: Appender |
| output: Outputter |
| |
| def text_accumulator() -> TextAccumulator: |
| """ |
| Creates a simple text accumulator / joiner. |
| |
| Returns a pair of callables: |
| append, output |
| "append" appends a string to the accumulator. |
| "output" returns the contents of the accumulator |
| joined together (''.join(accumulator)) and |
| empties the accumulator. |
| """ |
| text, append, output = _text_accumulator() |
| return TextAccumulator(append, output) |
| |
| @overload |
| def warn_or_fail( |
| *args: object, |
| fail: Literal[True], |
| filename: str | None = None, |
| line_number: int | None = None, |
| ) -> NoReturn: ... |
| |
| @overload |
| def warn_or_fail( |
| *args: object, |
| fail: Literal[False] = False, |
| filename: str | None = None, |
| line_number: int | None = None, |
| ) -> None: ... |
| |
| def warn_or_fail( |
| *args: object, |
| fail: bool = False, |
| filename: str | None = None, |
| line_number: int | None = None, |
| ) -> None: |
| joined = " ".join([str(a) for a in args]) |
| add, output = text_accumulator() |
| if fail: |
| add("Error") |
| else: |
| add("Warning") |
| if clinic: |
| if filename is None: |
| filename = clinic.filename |
| if getattr(clinic, 'block_parser', None) and (line_number is None): |
| line_number = clinic.block_parser.line_number |
| if filename is not None: |
| add(' in file "' + filename + '"') |
| if line_number is not None: |
| add(" on line " + str(line_number)) |
| add(':\n') |
| add(joined) |
| print(output()) |
| if fail: |
| sys.exit(-1) |
| |
| |
| def warn( |
| *args: object, |
| filename: str | None = None, |
| line_number: int | None = None, |
| ) -> None: |
| return warn_or_fail(*args, filename=filename, line_number=line_number, fail=False) |
| |
| def fail( |
| *args: object, |
| filename: str | None = None, |
| line_number: int | None = None, |
| ) -> NoReturn: |
| warn_or_fail(*args, filename=filename, line_number=line_number, fail=True) |
| |
| |
| def quoted_for_c_string(s: str) -> str: |
| for old, new in ( |
| ('\\', '\\\\'), # must be first! |
| ('"', '\\"'), |
| ("'", "\\'"), |
| ): |
| s = s.replace(old, new) |
| return s |
| |
| def c_repr(s: str) -> str: |
| return '"' + s + '"' |
| |
| |
| is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match |
| |
| def is_legal_py_identifier(s: str) -> bool: |
| return all(is_legal_c_identifier(field) for field in s.split('.')) |
| |
| # identifiers that are okay in Python but aren't a good idea in C. |
| # so if they're used Argument Clinic will add "_value" to the end |
| # of the name in C. |
| c_keywords = set(""" |
| asm auto break case char const continue default do double |
| else enum extern float for goto if inline int long |
| register return short signed sizeof static struct switch |
| typedef typeof union unsigned void volatile while |
| """.strip().split()) |
| |
| def ensure_legal_c_identifier(s: str) -> str: |
| # for now, just complain if what we're given isn't legal |
| if not is_legal_c_identifier(s): |
| fail("Illegal C identifier:", s) |
| # but if we picked a C keyword, pick something else |
| if s in c_keywords: |
| return s + "_value" |
| return s |
| |
| def rstrip_lines(s: str) -> str: |
| text, add, output = _text_accumulator() |
| for line in s.split('\n'): |
| add(line.rstrip()) |
| add('\n') |
| text.pop() |
| return output() |
| |
| def format_escape(s: str) -> str: |
| # double up curly-braces, this string will be used |
| # as part of a format_map() template later |
| s = s.replace('{', '{{') |
| s = s.replace('}', '}}') |
| return s |
| |
| def linear_format(s: str, **kwargs: str) -> str: |
| """ |
| Perform str.format-like substitution, except: |
| * The strings substituted must be on lines by |
| themselves. (This line is the "source line".) |
| * If the substitution text is empty, the source line |
| is removed in the output. |
| * If the field is not recognized, the original line |
| is passed unmodified through to the output. |
| * If the substitution text is not empty: |
| * Each line of the substituted text is indented |
| by the indent of the source line. |
| * A newline will be added to the end. |
| """ |
| |
| add, output = text_accumulator() |
| for line in s.split('\n'): |
| indent, curly, trailing = line.partition('{') |
| if not curly: |
| add(line) |
| add('\n') |
| continue |
| |
| name, curly, trailing = trailing.partition('}') |
| if not curly or name not in kwargs: |
| add(line) |
| add('\n') |
| continue |
| |
| if trailing: |
| fail("Text found after {" + name + "} block marker! It must be on a line by itself.") |
| if indent.strip(): |
| fail("Non-whitespace characters found before {" + name + "} block marker! It must be on a line by itself.") |
| |
| value = kwargs[name] |
| if not value: |
| continue |
| |
| value = textwrap.indent(rstrip_lines(value), indent) |
| add(value) |
| add('\n') |
| |
| return output()[:-1] |
| |
| def indent_all_lines(s: str, prefix: str) -> str: |
| """ |
| Returns 's', with 'prefix' prepended to all lines. |
| |
| If the last line is empty, prefix is not prepended |
| to it. (If s is blank, returns s unchanged.) |
| |
| (textwrap.indent only adds to non-blank lines.) |
| """ |
| split = s.split('\n') |
| last = split.pop() |
| final = [] |
| for line in split: |
| final.append(prefix) |
| final.append(line) |
| final.append('\n') |
| if last: |
| final.append(prefix) |
| final.append(last) |
| return ''.join(final) |
| |
| def suffix_all_lines(s: str, suffix: str) -> str: |
| """ |
| Returns 's', with 'suffix' appended to all lines. |
| |
| If the last line is empty, suffix is not appended |
| to it. (If s is blank, returns s unchanged.) |
| """ |
| split = s.split('\n') |
| last = split.pop() |
| final = [] |
| for line in split: |
| final.append(line) |
| final.append(suffix) |
| final.append('\n') |
| if last: |
| final.append(last) |
| final.append(suffix) |
| return ''.join(final) |
| |
| |
| def version_splitter(s: str) -> tuple[int, ...]: |
| """Splits a version string into a tuple of integers. |
| |
| The following ASCII characters are allowed, and employ |
| the following conversions: |
| a -> -3 |
| b -> -2 |
| c -> -1 |
| (This permits Python-style version strings such as "1.4b3".) |
| """ |
| version: list[int] = [] |
| accumulator: list[str] = [] |
| def flush() -> None: |
| if not accumulator: |
| raise ValueError('Unsupported version string: ' + repr(s)) |
| version.append(int(''.join(accumulator))) |
| accumulator.clear() |
| |
| for c in s: |
| if c.isdigit(): |
| accumulator.append(c) |
| elif c == '.': |
| flush() |
| elif c in 'abc': |
| flush() |
| version.append('abc'.index(c) - 3) |
| else: |
| raise ValueError('Illegal character ' + repr(c) + ' in version string ' + repr(s)) |
| flush() |
| return tuple(version) |
| |
| def version_comparitor(version1: str, version2: str) -> Literal[-1, 0, 1]: |
| iterator = itertools.zip_longest(version_splitter(version1), version_splitter(version2), fillvalue=0) |
| for i, (a, b) in enumerate(iterator): |
| if a < b: |
| return -1 |
| if a > b: |
| return 1 |
| return 0 |
| |
| |
| class CRenderData: |
| def __init__(self): |
| |
| # The C statements to declare variables. |
| # Should be full lines with \n eol characters. |
| self.declarations = [] |
| |
| # The C statements required to initialize the variables before the parse call. |
| # Should be full lines with \n eol characters. |
| self.initializers = [] |
| |
| # The C statements needed to dynamically modify the values |
| # parsed by the parse call, before calling the impl. |
| self.modifications = [] |
| |
| # The entries for the "keywords" array for PyArg_ParseTuple. |
| # Should be individual strings representing the names. |
| self.keywords = [] |
| |
| # The "format units" for PyArg_ParseTuple. |
| # Should be individual strings that will get |
| self.format_units = [] |
| |
| # The varargs arguments for PyArg_ParseTuple. |
| self.parse_arguments = [] |
| |
| # The parameter declarations for the impl function. |
| self.impl_parameters = [] |
| |
| # The arguments to the impl function at the time it's called. |
| self.impl_arguments = [] |
| |
| # For return converters: the name of the variable that |
| # should receive the value returned by the impl. |
| self.return_value = "return_value" |
| |
| # For return converters: the code to convert the return |
| # value from the parse function. This is also where |
| # you should check the _return_value for errors, and |
| # "goto exit" if there are any. |
| self.return_conversion = [] |
| self.converter_retval = "_return_value" |
| |
| # The C statements required to do some operations |
| # after the end of parsing but before cleaning up. |
| # These operations may be, for example, memory deallocations which |
| # can only be done without any error happening during argument parsing. |
| self.post_parsing = [] |
| |
| # The C statements required to clean up after the impl call. |
| self.cleanup = [] |
| |
| |
| class FormatCounterFormatter(string.Formatter): |
| """ |
| This counts how many instances of each formatter |
| "replacement string" appear in the format string. |
| |
| e.g. after evaluating "string {a}, {b}, {c}, {a}" |
| the counts dict would now look like |
| {'a': 2, 'b': 1, 'c': 1} |
| """ |
| def __init__(self): |
| self.counts = collections.Counter() |
| |
| def get_value(self, key, args, kwargs): |
| self.counts[key] += 1 |
| return '' |
| |
| class Language(metaclass=abc.ABCMeta): |
| |
| start_line = "" |
| body_prefix = "" |
| stop_line = "" |
| checksum_line = "" |
| |
| def __init__(self, filename): |
| pass |
| |
| @abc.abstractmethod |
| def render(self, clinic, signatures): |
| pass |
| |
| def parse_line(self, line): |
| pass |
| |
| def validate(self): |
| def assert_only_one(attr, *additional_fields): |
| """ |
| Ensures that the string found at getattr(self, attr) |
| contains exactly one formatter replacement string for |
| each valid field. The list of valid fields is |
| ['dsl_name'] extended by additional_fields. |
| |
| e.g. |
| self.fmt = "{dsl_name} {a} {b}" |
| |
| # this passes |
| self.assert_only_one('fmt', 'a', 'b') |
| |
| # this fails, the format string has a {b} in it |
| self.assert_only_one('fmt', 'a') |
| |
| # this fails, the format string doesn't have a {c} in it |
| self.assert_only_one('fmt', 'a', 'b', 'c') |
| |
| # this fails, the format string has two {a}s in it, |
| # it must contain exactly one |
| self.fmt2 = '{dsl_name} {a} {a}' |
| self.assert_only_one('fmt2', 'a') |
| |
| """ |
| fields = ['dsl_name'] |
| fields.extend(additional_fields) |
| line = getattr(self, attr) |
| fcf = FormatCounterFormatter() |
| fcf.format(line) |
| def local_fail(should_be_there_but_isnt): |
| if should_be_there_but_isnt: |
| fail("{} {} must contain {{{}}} exactly once!".format( |
| self.__class__.__name__, attr, name)) |
| else: |
| fail("{} {} must not contain {{{}}}!".format( |
| self.__class__.__name__, attr, name)) |
| |
| for name, count in fcf.counts.items(): |
| if name in fields: |
| if count > 1: |
| local_fail(True) |
| else: |
| local_fail(False) |
| for name in fields: |
| if fcf.counts.get(name) != 1: |
| local_fail(True) |
| |
| assert_only_one('start_line') |
| assert_only_one('stop_line') |
| |
| field = "arguments" if "{arguments}" in self.checksum_line else "checksum" |
| assert_only_one('checksum_line', field) |
| |
| |
| |
| class PythonLanguage(Language): |
| |
| language = 'Python' |
| start_line = "#/*[{dsl_name} input]" |
| body_prefix = "#" |
| stop_line = "#[{dsl_name} start generated code]*/" |
| checksum_line = "#/*[{dsl_name} end generated code: {arguments}]*/" |
| |
| |
| def permute_left_option_groups(l): |
| """ |
| Given [1, 2, 3], should yield: |
| () |
| (3,) |
| (2, 3) |
| (1, 2, 3) |
| """ |
| yield tuple() |
| accumulator = [] |
| for group in reversed(l): |
| accumulator = list(group) + accumulator |
| yield tuple(accumulator) |
| |
| |
| def permute_right_option_groups(l): |
| """ |
| Given [1, 2, 3], should yield: |
| () |
| (1,) |
| (1, 2) |
| (1, 2, 3) |
| """ |
| yield tuple() |
| accumulator = [] |
| for group in l: |
| accumulator.extend(group) |
| yield tuple(accumulator) |
| |
| |
| def permute_optional_groups(left, required, right): |
| """ |
| Generator function that computes the set of acceptable |
| argument lists for the provided iterables of |
| argument groups. (Actually it generates a tuple of tuples.) |
| |
| Algorithm: prefer left options over right options. |
| |
| If required is empty, left must also be empty. |
| """ |
| required = tuple(required) |
| if not required: |
| if left: |
| raise ValueError("required is empty but left is not") |
| |
| accumulator = [] |
| counts = set() |
| for r in permute_right_option_groups(right): |
| for l in permute_left_option_groups(left): |
| t = l + required + r |
| if len(t) in counts: |
| continue |
| counts.add(len(t)) |
| accumulator.append(t) |
| |
| accumulator.sort(key=len) |
| return tuple(accumulator) |
| |
| |
| def strip_leading_and_trailing_blank_lines(s): |
| lines = s.rstrip().split('\n') |
| while lines: |
| line = lines[0] |
| if line.strip(): |
| break |
| del lines[0] |
| return '\n'.join(lines) |
| |
| @functools.lru_cache() |
| def normalize_snippet(s, *, indent=0): |
| """ |
| Reformats s: |
| * removes leading and trailing blank lines |
| * ensures that it does not end with a newline |
| * dedents so the first nonwhite character on any line is at column "indent" |
| """ |
| s = strip_leading_and_trailing_blank_lines(s) |
| s = textwrap.dedent(s) |
| if indent: |
| s = textwrap.indent(s, ' ' * indent) |
| return s |
| |
| |
| def declare_parser(f, *, hasformat=False): |
| """ |
| Generates the code template for a static local PyArg_Parser variable, |
| with an initializer. For core code (incl. builtin modules) the |
| kwtuple field is also statically initialized. Otherwise |
| it is initialized at runtime. |
| """ |
| if hasformat: |
| fname = '' |
| format_ = '.format = "{format_units}:{name}",' |
| else: |
| fname = '.fname = "{name}",' |
| format_ = '' |
| |
| num_keywords = len([ |
| p for p in f.parameters.values() |
| if not p.is_positional_only() and not p.is_vararg() |
| ]) |
| if num_keywords == 0: |
| declarations = """ |
| #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) |
| # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) |
| #else |
| # define KWTUPLE NULL |
| #endif |
| """ |
| else: |
| declarations = """ |
| #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) |
| |
| #define NUM_KEYWORDS %d |
| static struct {{ |
| PyGC_Head _this_is_not_used; |
| PyObject_VAR_HEAD |
| PyObject *ob_item[NUM_KEYWORDS]; |
| }} _kwtuple = {{ |
| .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) |
| .ob_item = {{ {keywords_py} }}, |
| }}; |
| #undef NUM_KEYWORDS |
| #define KWTUPLE (&_kwtuple.ob_base.ob_base) |
| |
| #else // !Py_BUILD_CORE |
| # define KWTUPLE NULL |
| #endif // !Py_BUILD_CORE |
| """ % num_keywords |
| |
| declarations += """ |
| static const char * const _keywords[] = {{{keywords_c} NULL}}; |
| static _PyArg_Parser _parser = {{ |
| .keywords = _keywords, |
| %s |
| .kwtuple = KWTUPLE, |
| }}; |
| #undef KWTUPLE |
| """ % (format_ or fname) |
| return normalize_snippet(declarations) |
| |
| |
| def wrap_declarations(text, length=78): |
| """ |
| A simple-minded text wrapper for C function declarations. |
| |
| It views a declaration line as looking like this: |
| xxxxxxxx(xxxxxxxxx,xxxxxxxxx) |
| If called with length=30, it would wrap that line into |
| xxxxxxxx(xxxxxxxxx, |
| xxxxxxxxx) |
| (If the declaration has zero or one parameters, this |
| function won't wrap it.) |
| |
| If this doesn't work properly, it's probably better to |
| start from scratch with a more sophisticated algorithm, |
| rather than try and improve/debug this dumb little function. |
| """ |
| lines = [] |
| for line in text.split('\n'): |
| prefix, _, after_l_paren = line.partition('(') |
| if not after_l_paren: |
| lines.append(line) |
| continue |
| parameters, _, after_r_paren = after_l_paren.partition(')') |
| if not _: |
| lines.append(line) |
| continue |
| if ',' not in parameters: |
| lines.append(line) |
| continue |
| parameters = [x.strip() + ", " for x in parameters.split(',')] |
| prefix += "(" |
| if len(prefix) < length: |
| spaces = " " * len(prefix) |
| else: |
| spaces = " " * 4 |
| |
| while parameters: |
| line = prefix |
| first = True |
| while parameters: |
| if (not first and |
| (len(line) + len(parameters[0]) > length)): |
| break |
| line += parameters.pop(0) |
| first = False |
| if not parameters: |
| line = line.rstrip(", ") + ")" + after_r_paren |
| lines.append(line.rstrip()) |
| prefix = spaces |
| return "\n".join(lines) |
| |
| |
| class CLanguage(Language): |
| |
| body_prefix = "#" |
| language = 'C' |
| start_line = "/*[{dsl_name} input]" |
| body_prefix = "" |
| stop_line = "[{dsl_name} start generated code]*/" |
| checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" |
| |
| def __init__(self, filename): |
| super().__init__(filename) |
| self.cpp = cpp.Monitor(filename) |
| self.cpp.fail = fail |
| |
| def parse_line(self, line): |
| self.cpp.writeline(line) |
| |
| def render(self, clinic, signatures): |
| function = None |
| for o in signatures: |
| if isinstance(o, Function): |
| if function: |
| fail("You may specify at most one function per block.\nFound a block containing at least two:\n\t" + repr(function) + " and " + repr(o)) |
| function = o |
| return self.render_function(clinic, function) |
| |
| def docstring_for_c_string(self, f): |
| if re.search(r'[^\x00-\x7F]', f.docstring): |
| warn("Non-ascii character appear in docstring.") |
| |
| text, add, output = _text_accumulator() |
| # turn docstring into a properly quoted C string |
| for line in f.docstring.split('\n'): |
| add('"') |
| add(quoted_for_c_string(line)) |
| add('\\n"\n') |
| |
| if text[-2] == sig_end_marker: |
| # If we only have a signature, add the blank line that the |
| # __text_signature__ getter expects to be there. |
| add('"\\n"') |
| else: |
| text.pop() |
| add('"') |
| return ''.join(text) |
| |
| def output_templates(self, f): |
| parameters = list(f.parameters.values()) |
| assert parameters |
| assert isinstance(parameters[0].converter, self_converter) |
| del parameters[0] |
| requires_defining_class = False |
| if parameters and isinstance(parameters[0].converter, defining_class_converter): |
| requires_defining_class = True |
| del parameters[0] |
| converters = [p.converter for p in parameters] |
| |
| has_option_groups = parameters and (parameters[0].group or parameters[-1].group) |
| default_return_converter = (not f.return_converter or |
| f.return_converter.type == 'PyObject *') |
| |
| new_or_init = f.kind in (METHOD_NEW, METHOD_INIT) |
| |
| vararg = NO_VARARG |
| pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 |
| for i, p in enumerate(parameters, 1): |
| if p.is_keyword_only(): |
| assert not p.is_positional_only() |
| if not p.is_optional(): |
| min_kw_only = i - max_pos |
| elif p.is_vararg(): |
| if vararg != NO_VARARG: |
| fail("Too many var args") |
| pseudo_args += 1 |
| vararg = i - 1 |
| else: |
| if vararg == NO_VARARG: |
| max_pos = i |
| if p.is_positional_only(): |
| pos_only = i |
| if not p.is_optional(): |
| min_pos = i |
| |
| meth_o = (len(parameters) == 1 and |
| parameters[0].is_positional_only() and |
| not converters[0].is_optional() and |
| not requires_defining_class and |
| not new_or_init) |
| |
| # we have to set these things before we're done: |
| # |
| # docstring_prototype |
| # docstring_definition |
| # impl_prototype |
| # methoddef_define |
| # parser_prototype |
| # parser_definition |
| # impl_definition |
| # cpp_if |
| # cpp_endif |
| # methoddef_ifndef |
| |
| return_value_declaration = "PyObject *return_value = NULL;" |
| |
| methoddef_define = normalize_snippet(""" |
| #define {methoddef_name} \\ |
| {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, |
| """) |
| if new_or_init and not f.docstring: |
| docstring_prototype = docstring_definition = '' |
| else: |
| docstring_prototype = normalize_snippet(""" |
| PyDoc_VAR({c_basename}__doc__); |
| """) |
| docstring_definition = normalize_snippet(""" |
| PyDoc_STRVAR({c_basename}__doc__, |
| {docstring}); |
| """) |
| impl_definition = normalize_snippet(""" |
| static {impl_return_type} |
| {c_basename}_impl({impl_parameters}) |
| """) |
| impl_prototype = parser_prototype = parser_definition = None |
| |
| parser_prototype_keyword = normalize_snippet(""" |
| static PyObject * |
| {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) |
| """) |
| |
| parser_prototype_varargs = normalize_snippet(""" |
| static PyObject * |
| {c_basename}({self_type}{self_name}, PyObject *args) |
| """) |
| |
| parser_prototype_fastcall = normalize_snippet(""" |
| static PyObject * |
| {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) |
| """) |
| |
| parser_prototype_fastcall_keywords = normalize_snippet(""" |
| static PyObject * |
| {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) |
| """) |
| |
| parser_prototype_def_class = normalize_snippet(""" |
| static PyObject * |
| {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) |
| """) |
| |
| # parser_body_fields remembers the fields passed in to the |
| # previous call to parser_body. this is used for an awful hack. |
| parser_body_fields = () |
| def parser_body(prototype, *fields, declarations=''): |
| nonlocal parser_body_fields |
| add, output = text_accumulator() |
| add(prototype) |
| parser_body_fields = fields |
| |
| fields = list(fields) |
| fields.insert(0, normalize_snippet(""" |
| {{ |
| {return_value_declaration} |
| {parser_declarations} |
| {declarations} |
| {initializers} |
| """) + "\n") |
| # just imagine--your code is here in the middle |
| fields.append(normalize_snippet(""" |
| {modifications} |
| {return_value} = {c_basename}_impl({impl_arguments}); |
| {return_conversion} |
| {post_parsing} |
| |
| {exit_label} |
| {cleanup} |
| return return_value; |
| }} |
| """)) |
| for field in fields: |
| add('\n') |
| add(field) |
| return linear_format(output(), parser_declarations=declarations) |
| |
| if not parameters: |
| if not requires_defining_class: |
| # no parameters, METH_NOARGS |
| flags = "METH_NOARGS" |
| |
| parser_prototype = normalize_snippet(""" |
| static PyObject * |
| {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) |
| """) |
| parser_code = [] |
| |
| else: |
| assert not new_or_init |
| |
| flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" |
| |
| parser_prototype = parser_prototype_def_class |
| return_error = ('return NULL;' if default_return_converter |
| else 'goto exit;') |
| parser_code = [normalize_snippet(""" |
| if (nargs) {{ |
| PyErr_SetString(PyExc_TypeError, "{name}() takes no arguments"); |
| %s |
| }} |
| """ % return_error, indent=4)] |
| |
| if default_return_converter: |
| parser_definition = '\n'.join([ |
| parser_prototype, |
| '{{', |
| *parser_code, |
| ' return {c_basename}_impl({impl_arguments});', |
| '}}']) |
| else: |
| parser_definition = parser_body(parser_prototype, *parser_code) |
| |
| elif meth_o: |
| flags = "METH_O" |
| |
| if (isinstance(converters[0], object_converter) and |
| converters[0].format_unit == 'O'): |
| meth_o_prototype = normalize_snippet(""" |
| static PyObject * |
| {c_basename}({impl_parameters}) |
| """) |
| |
| if default_return_converter: |
| # maps perfectly to METH_O, doesn't need a return converter. |
| # so we skip making a parse function |
| # and call directly into the impl function. |
| impl_prototype = parser_prototype = parser_definition = '' |
| impl_definition = meth_o_prototype |
| else: |
| # SLIGHT HACK |
| # use impl_parameters for the parser here! |
| parser_prototype = meth_o_prototype |
| parser_definition = parser_body(parser_prototype) |
| |
| else: |
| argname = 'arg' |
| if parameters[0].name == argname: |
| argname += '_' |
| parser_prototype = normalize_snippet(""" |
| static PyObject * |
| {c_basename}({self_type}{self_name}, PyObject *%s) |
| """ % argname) |
| |
| displayname = parameters[0].get_displayname(0) |
| parsearg = converters[0].parse_arg(argname, displayname) |
| if parsearg is None: |
| parsearg = """ |
| if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{ |
| goto exit; |
| }} |
| """ % argname |
| parser_definition = parser_body(parser_prototype, |
| normalize_snippet(parsearg, indent=4)) |
| |
| elif has_option_groups: |
| # positional parameters with option groups |
| # (we have to generate lots of PyArg_ParseTuple calls |
| # in a big switch statement) |
| |
| flags = "METH_VARARGS" |
| parser_prototype = parser_prototype_varargs |
| |
| parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') |
| |
| elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: |
| if not new_or_init: |
| # positional-only, but no option groups |
| # we only need one call to _PyArg_ParseStack |
| |
| flags = "METH_FASTCALL" |
| parser_prototype = parser_prototype_fastcall |
| nargs = 'nargs' |
| argname_fmt = 'args[%d]' |
| else: |
| # positional-only, but no option groups |
| # we only need one call to PyArg_ParseTuple |
| |
| flags = "METH_VARARGS" |
| parser_prototype = parser_prototype_varargs |
| nargs = 'PyTuple_GET_SIZE(args)' |
| argname_fmt = 'PyTuple_GET_ITEM(args, %d)' |
| |
| |
| left_args = f"{nargs} - {max_pos}" |
| max_args = NO_VARARG if (vararg != NO_VARARG) else max_pos |
| parser_code = [normalize_snippet(""" |
| if (!_PyArg_CheckPositional("{name}", %s, %d, %s)) {{ |
| goto exit; |
| }} |
| """ % (nargs, min_pos, max_args), indent=4)] |
| |
| has_optional = False |
| for i, p in enumerate(parameters): |
| displayname = p.get_displayname(i+1) |
| argname = argname_fmt % i |
| |
| if p.is_vararg(): |
| if not new_or_init: |
| parser_code.append(normalize_snippet(""" |
| %s = PyTuple_New(%s); |
| if (!%s) {{ |
| goto exit; |
| }} |
| for (Py_ssize_t i = 0; i < %s; ++i) {{ |
| PyTuple_SET_ITEM(%s, i, Py_NewRef(args[%d + i])); |
| }} |
| """ % ( |
| p.converter.parser_name, |
| left_args, |
| p.converter.parser_name, |
| left_args, |
| p.converter.parser_name, |
| max_pos |
| ), indent=4)) |
| else: |
| parser_code.append(normalize_snippet(""" |
| %s = PyTuple_GetSlice(%d, -1); |
| """ % ( |
| p.converter.parser_name, |
| max_pos |
| ), indent=4)) |
| continue |
| |
| parsearg = p.converter.parse_arg(argname, displayname) |
| if parsearg is None: |
| #print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr) |
| parser_code = None |
| break |
| if has_optional or p.is_optional(): |
| has_optional = True |
| parser_code.append(normalize_snippet(""" |
| if (%s < %d) {{ |
| goto skip_optional; |
| }} |
| """, indent=4) % (nargs, i + 1)) |
| parser_code.append(normalize_snippet(parsearg, indent=4)) |
| |
| if parser_code is not None: |
| if has_optional: |
| parser_code.append("skip_optional:") |
| else: |
| if not new_or_init: |
| parser_code = [normalize_snippet(""" |
| if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}", |
| {parse_arguments})) {{ |
| goto exit; |
| }} |
| """, indent=4)] |
| else: |
| parser_code = [normalize_snippet(""" |
| if (!PyArg_ParseTuple(args, "{format_units}:{name}", |
| {parse_arguments})) {{ |
| goto exit; |
| }} |
| """, indent=4)] |
| parser_definition = parser_body(parser_prototype, *parser_code) |
| |
| else: |
| has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) |
| if vararg == NO_VARARG: |
| args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( |
| min_pos, |
| max_pos, |
| min_kw_only |
| ) |
| nargs = "nargs" |
| else: |
| args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( |
| min_pos, |
| max_pos, |
| min_kw_only, |
| vararg |
| ) |
| nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" |
| if not new_or_init: |
| flags = "METH_FASTCALL|METH_KEYWORDS" |
| parser_prototype = parser_prototype_fastcall_keywords |
| argname_fmt = 'args[%d]' |
| declarations = declare_parser(f) |
| declarations += "\nPyObject *argsbuf[%s];" % len(converters) |
| if has_optional_kw: |
| declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) |
| parser_code = [normalize_snippet(""" |
| args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); |
| if (!args) {{ |
| goto exit; |
| }} |
| """ % args_declaration, indent=4)] |
| else: |
| # positional-or-keyword arguments |
| flags = "METH_VARARGS|METH_KEYWORDS" |
| parser_prototype = parser_prototype_keyword |
| argname_fmt = 'fastargs[%d]' |
| declarations = declare_parser(f) |
| declarations += "\nPyObject *argsbuf[%s];" % len(converters) |
| declarations += "\nPyObject * const *fastargs;" |
| declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" |
| if has_optional_kw: |
| declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) |
| parser_code = [normalize_snippet(""" |
| fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); |
| if (!fastargs) {{ |
| goto exit; |
| }} |
| """ % args_declaration, indent=4)] |
| |
| if requires_defining_class: |
| flags = 'METH_METHOD|' + flags |
| parser_prototype = parser_prototype_def_class |
| |
| add_label = None |
| for i, p in enumerate(parameters): |
| if isinstance(p.converter, defining_class_converter): |
| raise ValueError("defining_class should be the first " |
| "parameter (after self)") |
| displayname = p.get_displayname(i+1) |
| parsearg = p.converter.parse_arg(argname_fmt % i, displayname) |
| if parsearg is None: |
| #print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr) |
| parser_code = None |
| break |
| if add_label and (i == pos_only or i == max_pos): |
| parser_code.append("%s:" % add_label) |
| add_label = None |
| if not p.is_optional(): |
| parser_code.append(normalize_snippet(parsearg, indent=4)) |
| elif i < pos_only: |
| add_label = 'skip_optional_posonly' |
| parser_code.append(normalize_snippet(""" |
| if (nargs < %d) {{ |
| goto %s; |
| }} |
| """ % (i + 1, add_label), indent=4)) |
| if has_optional_kw: |
| parser_code.append(normalize_snippet(""" |
| noptargs--; |
| """, indent=4)) |
| parser_code.append(normalize_snippet(parsearg, indent=4)) |
| else: |
| if i < max_pos: |
| label = 'skip_optional_pos' |
| first_opt = max(min_pos, pos_only) |
| else: |
| label = 'skip_optional_kwonly' |
| first_opt = max_pos + min_kw_only |
| if vararg != NO_VARARG: |
| first_opt += 1 |
| if i == first_opt: |
| add_label = label |
| parser_code.append(normalize_snippet(""" |
| if (!noptargs) {{ |
| goto %s; |
| }} |
| """ % add_label, indent=4)) |
| if i + 1 == len(parameters): |
| parser_code.append(normalize_snippet(parsearg, indent=4)) |
| else: |
| add_label = label |
| parser_code.append(normalize_snippet(""" |
| if (%s) {{ |
| """ % (argname_fmt % i), indent=4)) |
| parser_code.append(normalize_snippet(parsearg, indent=8)) |
| parser_code.append(normalize_snippet(""" |
| if (!--noptargs) {{ |
| goto %s; |
| }} |
| }} |
| """ % add_label, indent=4)) |
| |
| if parser_code is not None: |
| if add_label: |
| parser_code.append("%s:" % add_label) |
| else: |
| declarations = declare_parser(f, hasformat=True) |
| if not new_or_init: |
| parser_code = [normalize_snippet(""" |
| if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma} |
| {parse_arguments})) {{ |
| goto exit; |
| }} |
| """, indent=4)] |
| else: |
| parser_code = [normalize_snippet(""" |
| if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, |
| {parse_arguments})) {{ |
| goto exit; |
| }} |
| """, indent=4)] |
| parser_definition = parser_body(parser_prototype, *parser_code, |
| declarations=declarations) |
| |
| |
| if new_or_init: |
| methoddef_define = '' |
| |
| if f.kind == METHOD_NEW: |
| parser_prototype = parser_prototype_keyword |
| else: |
| return_value_declaration = "int return_value = -1;" |
| parser_prototype = normalize_snippet(""" |
| static int |
| {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) |
| """) |
| |
| fields = list(parser_body_fields) |
| parses_positional = 'METH_NOARGS' not in flags |
| parses_keywords = 'METH_KEYWORDS' in flags |
| if parses_keywords: |
| assert parses_positional |
| |
| if requires_defining_class: |
| raise ValueError("Slot methods cannot access their defining class.") |
| |
| if not parses_keywords: |
| declarations = '{base_type_ptr}' |
| fields.insert(0, normalize_snippet(""" |
| if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{ |
| goto exit; |
| }} |
| """, indent=4)) |
| if not parses_positional: |
| fields.insert(0, normalize_snippet(""" |
| if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{ |
| goto exit; |
| }} |
| """, indent=4)) |
| |
| parser_definition = parser_body(parser_prototype, *fields, |
| declarations=declarations) |
| |
| |
| if flags in ('METH_NOARGS', 'METH_O', 'METH_VARARGS'): |
| methoddef_cast = "(PyCFunction)" |
| methoddef_cast_end = "" |
| else: |
| methoddef_cast = "_PyCFunction_CAST(" |
| methoddef_cast_end = ")" |
| |
| if f.methoddef_flags: |
| flags += '|' + f.methoddef_flags |
| |
| methoddef_define = methoddef_define.replace('{methoddef_flags}', flags) |
| methoddef_define = methoddef_define.replace('{methoddef_cast}', methoddef_cast) |
| methoddef_define = methoddef_define.replace('{methoddef_cast_end}', methoddef_cast_end) |
| |
| methoddef_ifndef = '' |
| conditional = self.cpp.condition() |
| if not conditional: |
| cpp_if = cpp_endif = '' |
| else: |
| cpp_if = "#if " + conditional |
| cpp_endif = "#endif /* " + conditional + " */" |
| |
| if methoddef_define and f.full_name not in clinic.ifndef_symbols: |
| clinic.ifndef_symbols.add(f.full_name) |
| methoddef_ifndef = normalize_snippet(""" |
| #ifndef {methoddef_name} |
| #define {methoddef_name} |
| #endif /* !defined({methoddef_name}) */ |
| """) |
| |
| |
| # add ';' to the end of parser_prototype and impl_prototype |
| # (they mustn't be None, but they could be an empty string.) |
| assert parser_prototype is not None |
| if parser_prototype: |
| assert not parser_prototype.endswith(';') |
| parser_prototype += ';' |
| |
| if impl_prototype is None: |
| impl_prototype = impl_definition |
| if impl_prototype: |
| impl_prototype += ";" |
| |
| parser_definition = parser_definition.replace("{return_value_declaration}", return_value_declaration) |
| |
| d = { |
| "docstring_prototype" : docstring_prototype, |
| "docstring_definition" : docstring_definition, |
| "impl_prototype" : impl_prototype, |
| "methoddef_define" : methoddef_define, |
| "parser_prototype" : parser_prototype, |
| "parser_definition" : parser_definition, |
| "impl_definition" : impl_definition, |
| "cpp_if" : cpp_if, |
| "cpp_endif" : cpp_endif, |
| "methoddef_ifndef" : methoddef_ifndef, |
| } |
| |
| # make sure we didn't forget to assign something, |
| # and wrap each non-empty value in \n's |
| d2 = {} |
| for name, value in d.items(): |
| assert value is not None, "got a None value for template " + repr(name) |
| if value: |
| value = '\n' + value + '\n' |
| d2[name] = value |
| return d2 |
| |
| @staticmethod |
| def group_to_variable_name(group): |
| adjective = "left_" if group < 0 else "right_" |
| return "group_" + adjective + str(abs(group)) |
| |
| def render_option_group_parsing(self, f, template_dict): |
| # positional only, grouped, optional arguments! |
| # can be optional on the left or right. |
| # here's an example: |
| # |
| # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] |
| # |
| # Here group D are required, and all other groups are optional. |
| # (Group D's "group" is actually None.) |
| # We can figure out which sets of arguments we have based on |
| # how many arguments are in the tuple. |
| # |
| # Note that you need to count up on both sides. For example, |
| # you could have groups C+D, or C+D+E, or C+D+E+F. |
| # |
| # What if the number of arguments leads us to an ambiguous result? |
| # Clinic prefers groups on the left. So in the above example, |
| # five arguments would map to B+C, not C+D. |
| |
| add, output = text_accumulator() |
| parameters = list(f.parameters.values()) |
| if isinstance(parameters[0].converter, self_converter): |
| del parameters[0] |
| |
| group = None |
| left = [] |
| right = [] |
| required = [] |
| last = unspecified |
| |
| for p in parameters: |
| group_id = p.group |
| if group_id != last: |
| last = group_id |
| group = [] |
| if group_id < 0: |
| left.append(group) |
| elif group_id == 0: |
| group = required |
| else: |
| right.append(group) |
| group.append(p) |
| |
| count_min = sys.maxsize |
| count_max = -1 |
| |
| add("switch (PyTuple_GET_SIZE(args)) {\n") |
| for subset in permute_optional_groups(left, required, right): |
| count = len(subset) |
| count_min = min(count_min, count) |
| count_max = max(count_max, count) |
| |
| if count == 0: |
| add(""" case 0: |
| break; |
| """) |
| continue |
| |
| group_ids = {p.group for p in subset} # eliminate duplicates |
| d = {} |
| d['count'] = count |
| d['name'] = f.name |
| d['format_units'] = "".join(p.converter.format_unit for p in subset) |
| |
| parse_arguments = [] |
| for p in subset: |
| p.converter.parse_argument(parse_arguments) |
| d['parse_arguments'] = ", ".join(parse_arguments) |
| |
| group_ids.discard(0) |
| lines = [self.group_to_variable_name(g) + " = 1;" for g in group_ids] |
| lines = "\n".join(lines) |
| |
| s = """\ |
| case {count}: |
| if (!PyArg_ParseTuple(args, "{format_units}:{name}", {parse_arguments})) {{ |
| goto exit; |
| }} |
| {group_booleans} |
| break; |
| """ |
| s = linear_format(s, group_booleans=lines) |
| s = s.format_map(d) |
| add(s) |
| |
| add(" default:\n") |
| s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' |
| add(s.format(f.full_name, count_min, count_max)) |
| add(' goto exit;\n') |
| add("}") |
| template_dict['option_group_parsing'] = format_escape(output()) |
| |
| def render_function(self, clinic, f): |
| if not f: |
| return "" |
| |
| add, output = text_accumulator() |
| data = CRenderData() |
| |
| assert f.parameters, "We should always have a 'self' at this point!" |
| parameters = f.render_parameters |
| converters = [p.converter for p in parameters] |
| |
| templates = self.output_templates(f) |
| |
| f_self = parameters[0] |
| selfless = parameters[1:] |
| assert isinstance(f_self.converter, self_converter), "No self parameter in " + repr(f.full_name) + "!" |
| |
| last_group = 0 |
| first_optional = len(selfless) |
| positional = selfless and selfless[-1].is_positional_only() |
| new_or_init = f.kind in (METHOD_NEW, METHOD_INIT) |
| has_option_groups = False |
| |
| # offset i by -1 because first_optional needs to ignore self |
| for i, p in enumerate(parameters, -1): |
| c = p.converter |
| |
| if (i != -1) and (p.default is not unspecified): |
| first_optional = min(first_optional, i) |
| |
| if p.is_vararg(): |
| data.cleanup.append(f"Py_XDECREF({c.parser_name});") |
| |
| # insert group variable |
| group = p.group |
| if last_group != group: |
| last_group = group |
| if group: |
| group_name = self.group_to_variable_name(group) |
| data.impl_arguments.append(group_name) |
| data.declarations.append("int " + group_name + " = 0;") |
| data.impl_parameters.append("int " + group_name) |
| has_option_groups = True |
| |
| c.render(p, data) |
| |
| if has_option_groups and (not positional): |
| fail("You cannot use optional groups ('[' and ']')\nunless all parameters are positional-only ('/').") |
| |
| # HACK |
| # when we're METH_O, but have a custom return converter, |
| # we use "impl_parameters" for the parsing function |
| # because that works better. but that means we must |
| # suppress actually declaring the impl's parameters |
| # as variables in the parsing function. but since it's |
| # METH_O, we have exactly one anyway, so we know exactly |
| # where it is. |
| if ("METH_O" in templates['methoddef_define'] and |
| '{impl_parameters}' in templates['parser_prototype']): |
| data.declarations.pop(0) |
| |
| template_dict = {} |
| |
| full_name = f.full_name |
| template_dict['full_name'] = full_name |
| |
| if new_or_init: |
| name = f.cls.name |
| else: |
| name = f.name |
| |
| template_dict['name'] = name |
| |
| if f.c_basename: |
| c_basename = f.c_basename |
| else: |
| fields = full_name.split(".") |
| if fields[-1] == '__new__': |
| fields.pop() |
| c_basename = "_".join(fields) |
| |
| template_dict['c_basename'] = c_basename |
| |
| template_dict['methoddef_name'] = c_basename.upper() + "_METHODDEF" |
| |
| template_dict['docstring'] = self.docstring_for_c_string(f) |
| |
| template_dict['self_name'] = template_dict['self_type'] = template_dict['self_type_check'] = '' |
| for converter in converters: |
| converter.set_template_dict(template_dict) |
| |
| f.return_converter.render(f, data) |
| template_dict['impl_return_type'] = f.return_converter.type |
| |
| template_dict['declarations'] = format_escape("\n".join(data.declarations)) |
| template_dict['initializers'] = "\n\n".join(data.initializers) |
| template_dict['modifications'] = '\n\n'.join(data.modifications) |
| template_dict['keywords_c'] = ' '.join('"' + k + '",' |
| for k in data.keywords) |
| keywords = [k for k in data.keywords if k] |
| template_dict['keywords_py'] = ' '.join('&_Py_ID(' + k + '),' |
| for k in keywords) |
| template_dict['format_units'] = ''.join(data.format_units) |
| template_dict['parse_arguments'] = ', '.join(data.parse_arguments) |
| if data.parse_arguments: |
| template_dict['parse_arguments_comma'] = ','; |
| else: |
| template_dict['parse_arguments_comma'] = ''; |
| template_dict['impl_parameters'] = ", ".join(data.impl_parameters) |
| template_dict['impl_arguments'] = ", ".join(data.impl_arguments) |
| template_dict['return_conversion'] = format_escape("".join(data.return_conversion).rstrip()) |
| template_dict['post_parsing'] = format_escape("".join(data.post_parsing).rstrip()) |
| template_dict['cleanup'] = format_escape("".join(data.cleanup)) |
| template_dict['return_value'] = data.return_value |
| |
| # used by unpack tuple code generator |
| unpack_min = first_optional |
| unpack_max = len(selfless) |
| template_dict['unpack_min'] = str(unpack_min) |
| template_dict['unpack_max'] = str(unpack_max) |
| |
| if has_option_groups: |
| self.render_option_group_parsing(f, template_dict) |
| |
| # buffers, not destination |
| for name, destination in clinic.destination_buffers.items(): |
| template = templates[name] |
| if has_option_groups: |
| template = linear_format(template, |
| option_group_parsing=template_dict['option_group_parsing']) |
| template = linear_format(template, |
| declarations=template_dict['declarations'], |
| return_conversion=template_dict['return_conversion'], |
| initializers=template_dict['initializers'], |
| modifications=template_dict['modifications'], |
| post_parsing=template_dict['post_parsing'], |
| cleanup=template_dict['cleanup'], |
| ) |
| |
| # Only generate the "exit:" label |
| # if we have any gotos |
| need_exit_label = "goto exit;" in template |
| template = linear_format(template, |
| exit_label="exit:" if need_exit_label else '' |
| ) |
| |
| s = template.format_map(template_dict) |
| |
| # mild hack: |
| # reflow long impl declarations |
| if name in {"impl_prototype", "impl_definition"}: |
| s = wrap_declarations(s) |
| |
| if clinic.line_prefix: |
| s = indent_all_lines(s, clinic.line_prefix) |
| if clinic.line_suffix: |
| s = suffix_all_lines(s, clinic.line_suffix) |
| |
| destination.append(s) |
| |
| return clinic.get_destination('block').dump() |
| |
| |
| |
| |
| @contextlib.contextmanager |
| def OverrideStdioWith(stdout): |
| saved_stdout = sys.stdout |
| sys.stdout = stdout |
| try: |
| yield |
| finally: |
| assert sys.stdout is stdout |
| sys.stdout = saved_stdout |
| |
| |
| def create_regex(before, after, word=True, whole_line=True): |
| """Create an re object for matching marker lines.""" |
| group_re = r"\w+" if word else ".+" |
| pattern = r'{}({}){}' |
| if whole_line: |
| pattern = '^' + pattern + '$' |
| pattern = pattern.format(re.escape(before), group_re, re.escape(after)) |
| return re.compile(pattern) |
| |
| |
| class Block: |
| r""" |
| Represents a single block of text embedded in |
| another file. If dsl_name is None, the block represents |
| verbatim text, raw original text from the file, in |
| which case "input" will be the only non-false member. |
| If dsl_name is not None, the block represents a Clinic |
| block. |
| |
| input is always str, with embedded \n characters. |
| input represents the original text from the file; |
| if it's a Clinic block, it is the original text with |
| the body_prefix and redundant leading whitespace removed. |
| |
| dsl_name is either str or None. If str, it's the text |
| found on the start line of the block between the square |
| brackets. |
| |
| signatures is either list or None. If it's a list, |
| it may only contain clinic.Module, clinic.Class, and |
| clinic.Function objects. At the moment it should |
| contain at most one of each. |
| |
| output is either str or None. If str, it's the output |
| from this block, with embedded '\n' characters. |
| |
| indent is either str or None. It's the leading whitespace |
| that was found on every line of input. (If body_prefix is |
| not empty, this is the indent *after* removing the |
| body_prefix.) |
| |
| preindent is either str or None. It's the whitespace that |
| was found in front of every line of input *before* the |
| "body_prefix" (see the Language object). If body_prefix |
| is empty, preindent must always be empty too. |
| |
| To illustrate indent and preindent: Assume that '_' |
| represents whitespace. If the block processed was in a |
| Python file, and looked like this: |
| ____#/*[python] |
| ____#__for a in range(20): |
| ____#____print(a) |
| ____#[python]*/ |
| "preindent" would be "____" and "indent" would be "__". |
| |
| """ |
| def __init__(self, input, dsl_name=None, signatures=None, output=None, indent='', preindent=''): |
| assert isinstance(input, str) |
| self.input = input |
| self.dsl_name = dsl_name |
| self.signatures = signatures or [] |
| self.output = output |
| self.indent = indent |
| self.preindent = preindent |
| |
| def __repr__(self): |
| dsl_name = self.dsl_name or "text" |
| def summarize(s): |
| s = repr(s) |
| if len(s) > 30: |
| return s[:26] + "..." + s[0] |
| return s |
| return "".join(( |
| "<Block ", dsl_name, " input=", summarize(self.input), " output=", summarize(self.output), ">")) |
| |
| |
| class BlockParser: |
| """ |
| Block-oriented parser for Argument Clinic. |
| Iterator, yields Block objects. |
| """ |
| |
| def __init__(self, input, language, *, verify=True): |
| """ |
| "input" should be a str object |
| with embedded \n characters. |
| |
| "language" should be a Language object. |
| """ |
| language.validate() |
| |
| self.input = collections.deque(reversed(input.splitlines(keepends=True))) |
| self.block_start_line_number = self.line_number = 0 |
| |
| self.language = language |
| before, _, after = language.start_line.partition('{dsl_name}') |
| assert _ == '{dsl_name}' |
| self.find_start_re = create_regex(before, after, whole_line=False) |
| self.start_re = create_regex(before, after) |
| self.verify = verify |
| self.last_checksum_re = None |
| self.last_dsl_name = None |
| self.dsl_name = None |
| self.first_block = True |
| |
| def __iter__(self): |
| return self |
| |
| def __next__(self): |
| while True: |
| if not self.input: |
| raise StopIteration |
| |
| if self.dsl_name: |
| return_value = self.parse_clinic_block(self.dsl_name) |
| self.dsl_name = None |
| self.first_block = False |
| return return_value |
| block = self.parse_verbatim_block() |
| if self.first_block and not block.input: |
| continue |
| self.first_block = False |
| return block |
| |
| |
| def is_start_line(self, line): |
| match = self.start_re.match(line.lstrip()) |
| return match.group(1) if match else None |
| |
| def _line(self, lookahead=False): |
| self.line_number += 1 |
| line = self.input.pop() |
| if not lookahead: |
| self.language.parse_line(line) |
| return line |
| |
| def parse_verbatim_block(self): |
| add, output = text_accumulator() |
| self.block_start_line_number = self.line_number |
| |
| while self.input: |
| line = self._line() |
| dsl_name = self.is_start_line(line) |
| if dsl_name: |
| self.dsl_name = dsl_name |
| break |
| add(line) |
| |
| return Block(output()) |
| |
| def parse_clinic_block(self, dsl_name): |
| input_add, input_output = text_accumulator() |
| self.block_start_line_number = self.line_number + 1 |
| stop_line = self.language.stop_line.format(dsl_name=dsl_name) |
| body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) |
| |
| def is_stop_line(line): |
| # make sure to recognize stop line even if it |
| # doesn't end with EOL (it could be the very end of the file) |
| if line.startswith(stop_line): |
| remainder = line.removeprefix(stop_line) |
| if remainder and not remainder.isspace(): |
| fail(f"Garbage after stop line: {remainder!r}") |
| return True |
| else: |
| # gh-92256: don't allow incorrectly formatted stop lines |
| if line.lstrip().startswith(stop_line): |
| fail(f"Whitespace is not allowed before the stop line: {line!r}") |
| return False |
| |
| # consume body of program |
| while self.input: |
| line = self._line() |
| if is_stop_line(line) or self.is_start_line(line): |
| break |
| if body_prefix: |
| line = line.lstrip() |
| assert line.startswith(body_prefix) |
| line = line.removeprefix(body_prefix) |
| input_add(line) |
| |
| # consume output and checksum line, if present. |
| if self.last_dsl_name == dsl_name: |
| checksum_re = self.last_checksum_re |
| else: |
| before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}') |
| assert _ == '{arguments}' |
| checksum_re = create_regex(before, after, word=False) |
| self.last_dsl_name = dsl_name |
| self.last_checksum_re = checksum_re |
| |
| # scan forward for checksum line |
| output_add, output_output = text_accumulator() |
| arguments = None |
| while self.input: |
| line = self._line(lookahead=True) |
| match = checksum_re.match(line.lstrip()) |
| arguments = match.group(1) if match else None |
| if arguments: |
| break |
| output_add(line) |
| if self.is_start_line(line): |
| break |
| |
| output = output_output() |
| if arguments: |
| d = {} |
| for field in shlex.split(arguments): |
| name, equals, value = field.partition('=') |
| if not equals: |
| fail("Mangled Argument Clinic marker line:", repr(line)) |
| d[name.strip()] = value.strip() |
| |
| if self.verify: |
| if 'input' in d: |
| checksum = d['output'] |
| else: |
| checksum = d['checksum'] |
| |
| computed = compute_checksum(output, len(checksum)) |
| if checksum != computed: |
| fail("Checksum mismatch!\nExpected: {}\nComputed: {}\n" |
| "Suggested fix: remove all generated code including " |
| "the end marker,\n" |
| "or use the '-f' option." |
| .format(checksum, computed)) |
| else: |
| # put back output |
| output_lines = output.splitlines(keepends=True) |
| self.line_number -= len(output_lines) |
| self.input.extend(reversed(output_lines)) |
| output = None |
| |
| return Block(input_output(), dsl_name, output=output) |
| |
| |
| class BlockPrinter: |
| |
| def __init__(self, language, f=None): |
| self.language = language |
| self.f = f or io.StringIO() |
| |
| def print_block(self, block, *, core_includes=False): |
| input = block.input |
| output = block.output |
| dsl_name = block.dsl_name |
| write = self.f.write |
| |
| assert not ((dsl_name is None) ^ (output is None)), "you must specify dsl_name and output together, dsl_name " + repr(dsl_name) |
| |
| if not dsl_name: |
| write(input) |
| return |
| |
| write(self.language.start_line.format(dsl_name=dsl_name)) |
| write("\n") |
| |
| body_prefix = self.language.body_prefix.format(dsl_name=dsl_name) |
| if not body_prefix: |
| write(input) |
| else: |
| for line in input.split('\n'): |
| write(body_prefix) |
| write(line) |
| write("\n") |
| |
| write(self.language.stop_line.format(dsl_name=dsl_name)) |
| write("\n") |
| |
| output = '' |
| if core_includes: |
| output += textwrap.dedent(""" |
| #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) |
| # include "pycore_gc.h" // PyGC_Head |
| # include "pycore_runtime.h" // _Py_ID() |
| #endif |
| |
| """) |
| |
| input = ''.join(block.input) |
| output += ''.join(block.output) |
| if output: |
| if not output.endswith('\n'): |
| output += '\n' |
| write(output) |
| |
| arguments = "output={output} input={input}".format( |
| output=compute_checksum(output, 16), |
| input=compute_checksum(input, 16) |
| ) |
| write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) |
| write("\n") |
| |
| def write(self, text): |
| self.f.write(text) |
| |
| |
| class BufferSeries: |
| """ |
| Behaves like a "defaultlist". |
| When you ask for an index that doesn't exist yet, |
| the object grows the list until that item exists. |
| So o[n] will always work. |
| |
| Supports negative indices for actual items. |
| e.g. o[-1] is an element immediately preceding o[0]. |
| """ |
| |
| def __init__(self): |
| self._start = 0 |
| self._array = [] |
| self._constructor = _text_accumulator |
| |
| def __getitem__(self, i): |
| i -= self._start |
| if i < 0: |
| self._start += i |
| prefix = [self._constructor() for x in range(-i)] |
| self._array = prefix + self._array |
| i = 0 |
| while i >= len(self._array): |
| self._array.append(self._constructor()) |
| return self._array[i] |
| |
| def clear(self): |
| for ta in self._array: |
| ta._text.clear() |
| |
| def dump(self): |
| texts = [ta.output() for ta in self._array] |
| return "".join(texts) |
| |
| |
| class Destination: |
| def __init__(self, name, type, clinic, *args): |
| self.name = name |
| self.type = type |
| self.clinic = clinic |
| valid_types = ('buffer', 'file', 'suppress') |
| if type not in valid_types: |
| fail("Invalid destination type " + repr(type) + " for " + name + " , must be " + ', '.join(valid_types)) |
| extra_arguments = 1 if type == "file" else 0 |
| if len(args) < extra_arguments: |
| fail("Not enough arguments for destination " + name + " new " + type) |
| if len(args) > extra_arguments: |
| fail("Too many arguments for destination " + name + " new " + type) |
| if type =='file': |
| d = {} |
| filename = clinic.filename |
| d['path'] = filename |
| dirname, basename = os.path.split(filename) |
| if not dirname: |
| dirname = '.' |
| d['dirname'] = dirname |
| d['basename'] = basename |
| d['basename_root'], d['basename_extension'] = os.path.splitext(filename) |
| self.filename = args[0].format_map(d) |
| |
| self.buffers = BufferSeries() |
| |
| def __repr__(self): |
| if self.type == 'file': |
| file_repr = " " + repr(self.filename) |
| else: |
| file_repr = '' |
| return "".join(("<Destination ", self.name, " ", self.type, file_repr, ">")) |
| |
| def clear(self): |
| if self.type != 'buffer': |
| fail("Can't clear destination" + self.name + " , it's not of type buffer") |
| self.buffers.clear() |
| |
| def dump(self): |
| return self.buffers.dump() |
| |
| |
| # maps strings to Language objects. |
| # "languages" maps the name of the language ("C", "Python"). |
| # "extensions" maps the file extension ("c", "py"). |
| LangDict = dict[str, Callable[[str], Language]] |
| |
| languages = { 'C': CLanguage, 'Python': PythonLanguage } |
| extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } |
| extensions['py'] = PythonLanguage |
| |
| |
| def file_changed(filename: str, new_contents: str) -> bool: |
| """Return true if file contents changed (meaning we must update it)""" |
| try: |
| with open(filename, encoding="utf-8") as fp: |
| old_contents = fp.read() |
| return old_contents != new_contents |
| except FileNotFoundError: |
| return True |
| |
| |
| def write_file(filename: str, new_contents: str): |
| # Atomic write using a temporary file and os.replace() |
| filename_new = f"{filename}.new" |
| with open(filename_new, "w", encoding="utf-8") as fp: |
| fp.write(new_contents) |
| |
| try: |
| os.replace(filename_new, filename) |
| except: |
| os.unlink(filename_new) |
| raise |
| |
| |
| ClassDict = dict[str, "Class"] |
| DestinationDict = dict[str, Destination] |
| ModuleDict = dict[str, "Module"] |
| ParserDict = dict[str, "DSLParser"] |
| |
| clinic = None |
| class Clinic: |
| |
| presets_text = """ |
| preset block |
| everything block |
| methoddef_ifndef buffer 1 |
| docstring_prototype suppress |
| parser_prototype suppress |
| cpp_if suppress |
| cpp_endif suppress |
| |
| preset original |
| everything block |
| methoddef_ifndef buffer 1 |
| docstring_prototype suppress |
| parser_prototype suppress |
| cpp_if suppress |
| cpp_endif suppress |
| |
| preset file |
| everything file |
| methoddef_ifndef file 1 |
| docstring_prototype suppress |
| parser_prototype suppress |
| impl_definition block |
| |
| preset buffer |
| everything buffer |
| methoddef_ifndef buffer 1 |
| impl_definition block |
| docstring_prototype suppress |
| impl_prototype suppress |
| parser_prototype suppress |
| |
| preset partial-buffer |
| everything buffer |
| methoddef_ifndef buffer 1 |
| docstring_prototype block |
| impl_prototype suppress |
| methoddef_define block |
| parser_prototype block |
| impl_definition block |
| |
| """ |
| |
| def __init__( |
| self, |
| language: CLanguage, |
| printer: BlockPrinter | None = None, |
| *, |
| verify: bool = True, |
| filename: str | None = None |
| ) -> None: |
| # maps strings to Parser objects. |
| # (instantiated from the "parsers" global.) |
| self.parsers: ParserDict = {} |
| self.language: CLanguage = language |
| if printer: |
| fail("Custom printers are broken right now") |
| self.printer = printer or BlockPrinter(language) |
| self.verify = verify |
| self.filename = filename |
| self.modules: ModuleDict = {} |
| self.classes: ClassDict = {} |
| self.functions: list[Function] = [] |
| |
| self.line_prefix = self.line_suffix = '' |
| |
| self.destinations: DestinationDict = {} |
| self.add_destination("block", "buffer") |
| self.add_destination("suppress", "suppress") |
| self.add_destination("buffer", "buffer") |
| if filename: |
| self.add_destination("file", "file", "{dirname}/clinic/{basename}.h") |
| |
| d = self.get_destination_buffer |
| self.destination_buffers = { |
| 'cpp_if': d('file'), |
| 'docstring_prototype': d('suppress'), |
| 'docstring_definition': d('file'), |
| 'methoddef_define': d('file'), |
| 'impl_prototype': d('file'), |
| 'parser_prototype': d('suppress'), |
| 'parser_definition': d('file'), |
| 'cpp_endif': d('file'), |
| 'methoddef_ifndef': d('file', 1), |
| 'impl_definition': d('block'), |
| } |
| |
| DestBufferType = dict[str, Callable[..., Any]] |
| DestBufferList = list[DestBufferType] |
| |
| self.destination_buffers_stack: DestBufferList = [] |
| self.ifndef_symbols: set[str] = set() |
| |
| self.presets: dict[str, dict[Any, Any]] = {} |
| preset = None |
| for line in self.presets_text.strip().split('\n'): |
| line = line.strip() |
| if not line: |
| continue |
| name, value, *options = line.split() |
| if name == 'preset': |
| self.presets[value] = preset = {} |
| continue |
| |
| if len(options): |
| index = int(options[0]) |
| else: |
| index = 0 |
| buffer = self.get_destination_buffer(value, index) |
| |
| if name == 'everything': |
| for name in self.destination_buffers: |
| preset[name] = buffer |
| continue |
| |
| assert name in self.destination_buffers |
| preset[name] = buffer |
| |
| global clinic |
| clinic = self |
| |
| def add_destination( |
| self, |
| name: str, |
| type: str, |
| *args |
| ) -> None: |
| if name in self.destinations: |
| fail("Destination already exists: " + repr(name)) |
| self.destinations[name] = Destination(name, type, self, *args) |
| |
| def get_destination(self, name: str) -> Destination: |
| d = self.destinations.get(name) |
| if not d: |
| fail("Destination does not exist: " + repr(name)) |
| return d |
| |
| def get_destination_buffer( |
| self, |
| name: str, |
| item: int = 0 |
| ): |
| d = self.get_destination(name) |
| return d.buffers[item] |
| |
| def parse(self, input): |
| printer = self.printer |
| self.block_parser = BlockParser(input, self.language, verify=self.verify) |
| for block in self.block_parser: |
| dsl_name = block.dsl_name |
| if dsl_name: |
| if dsl_name not in self.parsers: |
| assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block." |
| self.parsers[dsl_name] = parsers[dsl_name](self) |
| parser = self.parsers[dsl_name] |
| try: |
| parser.parse(block) |
| except Exception: |
| fail('Exception raised during parsing:\n' + |
| traceback.format_exc().rstrip()) |
| printer.print_block(block) |
| |
| clinic_out = [] |
| |
| # these are destinations not buffers |
| for name, destination in self.destinations.items(): |
| if destination.type == 'suppress': |
| continue |
| output = destination.dump() |
| |
| if output: |
| |
| block = Block("", dsl_name="clinic", output=output) |
| |
| if destination.type == 'buffer': |
| block.input = "dump " + name + "\n" |
| warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.") |
| printer.write("\n") |
| printer.print_block(block) |
| continue |
| |
| if destination.type == 'file': |
| try: |
| dirname = os.path.dirname(destination.filename) |
| try: |
| os.makedirs(dirname) |
| except FileExistsError: |
| if not os.path.isdir(dirname): |
| fail("Can't write to destination {}, " |
| "can't make directory {}!".format( |
| destination.filename, dirname)) |
| if self.verify: |
| with open(destination.filename) as f: |
| parser_2 = BlockParser(f.read(), language=self.language) |
| blocks = list(parser_2) |
| if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): |
| fail("Modified destination file " + repr(destination.filename) + ", not overwriting!") |
| except FileNotFoundError: |
| pass |
| |
| block.input = 'preserve\n' |
| printer_2 = BlockPrinter(self.language) |
| printer_2.print_block(block, core_includes=True) |
| pair = destination.filename, printer_2.f.getvalue() |
| clinic_out.append(pair) |
| continue |
| |
| return printer.f.getvalue(), clinic_out |
| |
| |
| def _module_and_class(self, fields): |
| """ |
| fields should be an iterable of field names. |
| returns a tuple of (module, class). |
| the module object could actually be self (a clinic object). |
| this function is only ever used to find the parent of where |
| a new class/module should go. |
| """ |
| in_classes = False |
| parent = module = self |
| cls = None |
| so_far = [] |
| |
| for field in fields: |
| so_far.append(field) |
| if not in_classes: |
| child = parent.modules.get(field) |
| if child: |
| parent = module = child |
| continue |
| in_classes = True |
| if not hasattr(parent, 'classes'): |
| return module, cls |
| child = parent.classes.get(field) |
| if not child: |
| fail('Parent class or module ' + '.'.join(so_far) + " does not exist.") |
| cls = parent = child |
| |
| return module, cls |
| |
| |
| def parse_file( |
| filename: str, |
| *, |
| verify: bool = True, |
| output: str | None = None |
| ) -> None: |
| if not output: |
| output = filename |
| |
| extension = os.path.splitext(filename)[1][1:] |
| if not extension: |
| fail("Can't extract file type for file " + repr(filename)) |
| |
| try: |
| language = extensions[extension](filename) |
| except KeyError: |
| fail("Can't identify file type for file " + repr(filename)) |
| |
| with open(filename, encoding="utf-8") as f: |
| raw = f.read() |
| |
| # exit quickly if there are no clinic markers in the file |
| find_start_re = BlockParser("", language).find_start_re |
| if not find_start_re.search(raw): |
| return |
| |
| assert isinstance(language, CLanguage) |
| clinic = Clinic(language, verify=verify, filename=filename) |
| src_out, clinic_out = clinic.parse(raw) |
| |
| changes = [(fn, data) for fn, data in clinic_out if file_changed(fn, data)] |
| if changes: |
| # Always (re)write the source file. |
| write_file(output, src_out) |
| for fn, data in clinic_out: |
| write_file(fn, data) |
| |
| |
| def compute_checksum( |
| input: str | None, |
| length: int | None = None |
| ) -> str: |
| input = input or '' |
| s = hashlib.sha1(input.encode('utf-8')).hexdigest() |
| if length: |
| s = s[:length] |
| return s |
| |
| |
| |
| |
| class PythonParser: |
| def __init__(self, clinic: Clinic) -> None: |
| pass |
| |
| def parse(self, block: Block) -> None: |
| s = io.StringIO() |
| with OverrideStdioWith(s): |
| exec(block.input) |
| block.output = s.getvalue() |
| |
| |
| class Module: |
| def __init__( |
| self, |
| name: str, |
| module = None |
| ) -> None: |
| self.name = name |
| self.module = self.parent = module |
| |
| self.modules: ModuleDict = {} |
| self.classes: ClassDict = {} |
| self.functions: list[Function] = [] |
| |
| def __repr__(self) -> str: |
| return "<clinic.Module " + repr(self.name) + " at " + str(id(self)) + ">" |
| |
| |
| class Class: |
| def __init__( |
| self, |
| name: str, |
| module: Module | None = None, |
| cls = None, |
| typedef: str | None = None, |
| type_object: str | None = None |
| ) -> None: |
| self.name = name |
| self.module = module |
| self.cls = cls |
| self.typedef = typedef |
| self.type_object = type_object |
| self.parent = cls or module |
| |
| self.classes: ClassDict = {} |
| self.functions: list[Function] = [] |
| |
| def __repr__(self) -> str: |
| return "<clinic.Class " + repr(self.name) + " at " + str(id(self)) + ">" |
| |
| |
| unsupported_special_methods: set[str] = set(""" |
| |
| __abs__ |
| __add__ |
| __and__ |
| __call__ |
| __delitem__ |
| __divmod__ |
| __eq__ |
| __float__ |
| __floordiv__ |
| __ge__ |
| __getattr__ |
| __getattribute__ |
| __getitem__ |
| __gt__ |
| __hash__ |
| __iadd__ |
| __iand__ |
| __ifloordiv__ |
| __ilshift__ |
| __imatmul__ |
| __imod__ |
| __imul__ |
| __index__ |
| __int__ |
| __invert__ |
| __ior__ |
| __ipow__ |
| __irshift__ |
| __isub__ |
| __iter__ |
| __itruediv__ |
| __ixor__ |
| __le__ |
| __len__ |
| __lshift__ |
| __lt__ |
| __matmul__ |
| __mod__ |
| __mul__ |
| __neg__ |
| __next__ |
| __or__ |
| __pos__ |
| __pow__ |
| __radd__ |
| __rand__ |
| __rdivmod__ |
| __repr__ |
| __rfloordiv__ |
| __rlshift__ |
| __rmatmul__ |
| __rmod__ |
| __rmul__ |
| __ror__ |
| __rpow__ |
| __rrshift__ |
| __rshift__ |
| __rsub__ |
| __rtruediv__ |
| __rxor__ |
| __setattr__ |
| __setitem__ |
| __str__ |
| __sub__ |
| __truediv__ |
| __xor__ |
| |
| """.strip().split()) |
| |
| |
| INVALID, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW = """ |
| INVALID, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW |
| """.replace(",", "").strip().split() |
| |
| ParamDict = dict[str, "Parameter"] |
| ReturnConverterType = Callable[..., "CReturnConverter"] |
| |
| class Function: |
| """ |
| Mutable duck type for inspect.Function. |
| |
| docstring - a str containing |
| * embedded line breaks |
| * text outdented to the left margin |
| * no trailing whitespace. |
| It will always be true that |
| (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) |
| """ |
| |
| def __init__( |
| self, |
| parameters: ParamDict | None = None, |
| *, |
| name: str, |
| module: Module, |
| cls: Class | None = None, |
| c_basename: str | None = None, |
| full_name: str | None = None, |
| return_converter: ReturnConverterType, |
| return_annotation = inspect.Signature.empty, |
| docstring: str | None = None, |
| kind: str = CALLABLE, |
| coexist: bool = False, |
| docstring_only: bool = False |
| ) -> None: |
| self.parameters = parameters or {} |
| self.return_annotation = return_annotation |
| self.name = name |
| self.full_name = full_name |
| self.module = module |
| self.cls = cls |
| self.parent = cls or module |
| self.c_basename = c_basename |
| self.return_converter = return_converter |
| self.docstring = docstring or '' |
| self.kind = kind |
| self.coexist = coexist |
| self.self_converter = None |
| # docstring_only means "don't generate a machine-readable |
| # signature, just a normal docstring". it's True for |
| # functions with optional groups because we can't represent |
| # those accurately with inspect.Signature in 3.4. |
| self.docstring_only = docstring_only |
| |
| self.rendered_parameters = None |
| |
| __render_parameters__ = None |
| @property |
| def render_parameters(self): |
| if not self.__render_parameters__: |
| self.__render_parameters__ = l = [] |
| for p in self.parameters.values(): |
| p = p.copy() |
| p.converter.pre_render() |
| l.append(p) |
| return self.__render_parameters__ |
| |
| @property |
| def methoddef_flags(self) -> str | None: |
| if self.kind in (METHOD_INIT, METHOD_NEW): |
| return None |
| flags = [] |
| if self.kind == CLASS_METHOD: |
| flags.append('METH_CLASS') |
| elif self.kind == STATIC_METHOD: |
| flags.append('METH_STATIC') |
| else: |
| assert self.kind == CALLABLE, "unknown kind: " + repr(self.kind) |
| if self.coexist: |
| flags.append('METH_COEXIST') |
| return '|'.join(flags) |
| |
| def __repr__(self) -> str: |
| return '<clinic.Function ' + self.name + '>' |
| |
| def copy(self, **overrides) -> "Function": |
| kwargs = { |
| 'name': self.name, 'module': self.module, 'parameters': self.parameters, |
| 'cls': self.cls, 'c_basename': self.c_basename, |
| 'full_name': self.full_name, |
| 'return_converter': self.return_converter, 'return_annotation': self.return_annotation, |
| 'docstring': self.docstring, 'kind': self.kind, 'coexist': self.coexist, |
| 'docstring_only': self.docstring_only, |
| } |
| kwargs.update(overrides) |
| f = Function(**kwargs) |
| f.parameters = { |
| name: value.copy(function=f) |
| for name, value in f.parameters.items() |
| } |
| return f |
| |
| |
| class Parameter: |
| """ |
| Mutable duck type of inspect.Parameter. |
| """ |
| |
| def __init__( |
| self, |
| name: str, |
| kind: str, |
| *, |
| default = inspect.Parameter.empty, |
| function: Function, |
| converter: "CConverter", |
| annotation = inspect.Parameter.empty, |
| docstring: str | None = None, |
| group: int = 0 |
| ) -> None: |
| self.name = name |
| self.kind = kind |
| self.default = default |
| self.function = function |
| self.converter = converter |
| self.annotation = annotation |
| self.docstring = docstring or '' |
| self.group = group |
| |
| def __repr__(self) -> str: |
| return '<clinic.Parameter ' + self.name + '>' |
| |
| def is_keyword_only(self) -> bool: |
| return self.kind == inspect.Parameter.KEYWORD_ONLY |
| |
| def is_positional_only(self) -> bool: |
| return self.kind == inspect.Parameter.POSITIONAL_ONLY |
| |
| def is_vararg(self) -> bool: |
| return self.kind == inspect.Parameter.VAR_POSITIONAL |
| |
| def is_optional(self) -> bool: |
| return not self.is_vararg() and (self.default is not unspecified) |
| |
| def copy(self, **overrides) -> "Parameter": |
| kwargs = { |
| 'name': self.name, 'kind': self.kind, 'default':self.default, |
| 'function': self.function, 'converter': self.converter, 'annotation': self.annotation, |
| 'docstring': self.docstring, 'group': self.group, |
| } |
| kwargs.update(overrides) |
| if 'converter' not in overrides: |
| converter = copy.copy(self.converter) |
| converter.function = kwargs['function'] |
| kwargs['converter'] = converter |
| return Parameter(**kwargs) |
| |
| def get_displayname(self, i: int) -> str: |
| if i == 0: |
| return '"argument"' |
| if not self.is_positional_only(): |
| return f'"argument {self.name!r}"' |
| else: |
| return f'"argument {i}"' |
| |
| |
| class LandMine: |
| # try to access any |
| def __init__(self, message: str) -> None: |
| self.__message__ = message |
| |
| def __repr__(self) -> str: |
| return '<LandMine ' + repr(self.__message__) + ">" |
| |
| def __getattribute__(self, name: str): |
| if name in ('__repr__', '__message__'): |
| return super().__getattribute__(name) |
| # raise RuntimeError(repr(name)) |
| fail("Stepped on a land mine, trying to access attribute " + repr(name) + ":\n" + self.__message__) |
| |
| |
| def add_c_converter(f, name=None): |
| if not name: |
| name = f.__name__ |
| if not name.endswith('_converter'): |
| return f |
| name = name.removesuffix('_converter') |
| converters[name] = f |
| return f |
| |
| def add_default_legacy_c_converter(cls): |
| # automatically add converter for default format unit |
| # (but without stomping on the existing one if it's already |
| # set, in case you subclass) |
| if ((cls.format_unit not in ('O&', '')) and |
| (cls.format_unit not in legacy_converters)): |
| legacy_converters[cls.format_unit] = cls |
| return cls |
| |
| def add_legacy_c_converter(format_unit, **kwargs): |
| """ |
| Adds a legacy converter. |
| """ |
| def closure(f): |
| if not kwargs: |
| added_f = f |
| else: |
| added_f = functools.partial(f, **kwargs) |
| if format_unit: |
| legacy_converters[format_unit] = added_f |
| return f |
| return closure |
| |
| class CConverterAutoRegister(type): |
| def __init__(cls, name, bases, classdict): |
| add_c_converter(cls) |
| add_default_legacy_c_converter(cls) |
| |
| class CConverter(metaclass=CConverterAutoRegister): |
| """ |
| For the init function, self, name, function, and default |
| must be keyword-or-positional parameters. All other |
| parameters must be keyword-only. |
| """ |
| |
| # The C name to use for this variable. |
| name: str | None = None |
| |
| # The Python name to use for this variable. |
| py_name: str | None = None |
| |
| # The C type to use for this variable. |
| # 'type' should be a Python string specifying the type, e.g. "int". |
| # If this is a pointer type, the type string should end with ' *'. |
| type: str | None = None |
| |
| # The Python default value for this parameter, as a Python value. |
| # Or the magic value "unspecified" if there is no default. |
| # Or the magic value "unknown" if this value is a cannot be evaluated |
| # at Argument-Clinic-preprocessing time (but is presumed to be valid |
| # at runtime). |
| default: object = unspecified |
| |
| # If not None, default must be isinstance() of this type. |
| # (You can also specify a tuple of types.) |
| default_type: bltns.type[Any] | tuple[bltns.type[Any], ...] | None = None |
| |
| # "default" converted into a C value, as a string. |
| # Or None if there is no default. |
| c_default: str | None = None |
| |
| # "default" converted into a Python value, as a string. |
| # Or None if there is no default. |
| py_default: str | None = None |
| |
| # The default value used to initialize the C variable when |
| # there is no default, but not specifying a default may |
| # result in an "uninitialized variable" warning. This can |
| # easily happen when using option groups--although |
| # properly-written code won't actually use the variable, |
| # the variable does get passed in to the _impl. (Ah, if |
| # only dataflow analysis could inline the static function!) |
| # |
| # This value is specified as a string. |
| # Every non-abstract subclass should supply a valid value. |
| c_ignored_default: str = 'NULL' |
| |
| # If true, wrap with Py_UNUSED. |
| unused = False |
| |
| # The C converter *function* to be used, if any. |
| # (If this is not None, format_unit must be 'O&'.) |
| converter: str | None = None |
| |
| # Should Argument Clinic add a '&' before the name of |
| # the variable when passing it into the _impl function? |
| impl_by_reference = False |
| |
| # Should Argument Clinic add a '&' before the name of |
| # the variable when passing it into PyArg_ParseTuple (AndKeywords)? |
| parse_by_reference = True |
| |
| ############################################################# |
| ############################################################# |
| ## You shouldn't need to read anything below this point to ## |
| ## write your own converter functions. ## |
| ############################################################# |
| ############################################################# |
| |
| # The "format unit" to specify for this variable when |
| # parsing arguments using PyArg_ParseTuple (AndKeywords). |
| # Custom converters should always use the default value of 'O&'. |
| format_unit = 'O&' |
| |
| # What encoding do we want for this variable? Only used |
| # by format units starting with 'e'. |
| encoding: str | None = None |
| |
| # Should this object be required to be a subclass of a specific type? |
| # If not None, should be a string representing a pointer to a |
| # PyTypeObject (e.g. "&PyUnicode_Type"). |
| # Only used by the 'O!' format unit (and the "object" converter). |
| subclass_of = None |
| |
| # Do we want an adjacent '_length' variable for this variable? |
| # Only used by format units ending with '#'. |
| length = False |
| |
| # Should we show this parameter in the generated |
| # __text_signature__? This is *almost* always True. |
| # (It's only False for __new__, __init__, and METH_STATIC functions.) |
| show_in_signature = True |
| |
| # Overrides the name used in a text signature. |
| # The name used for a "self" parameter must be one of |
| # self, type, or module; however users can set their own. |
| # This lets the self_converter overrule the user-settable |
| # name, *just* for the text signature. |
| # Only set by self_converter. |
| signature_name = None |
| |
| # keep in sync with self_converter.__init__! |
| def __init__(self, |
| # Positional args: |
| name: str, |
| py_name: str, |
| function, |
| default: object = unspecified, |
| *, # Keyword only args: |
| c_default: str | None = None, |
| py_default: str | None = None, |
| annotation: str | Literal[Sentinels.unspecified] = unspecified, |
| unused: bool = False, |
| **kwargs |
| ): |
| self.name = ensure_legal_c_identifier(name) |
| self.py_name = py_name |
| self.unused = unused |
| |
| if default is not unspecified: |
| if (self.default_type |
| and default is not unknown |
| and not isinstance(default, self.default_type) |
| ): |
| if isinstance(self.default_type, type): |
| types_str = self.default_type.__name__ |
| else: |
| names = [cls.__name__ for cls in self.default_type] |
| types_str = ', '.join(names) |
| fail("{}: default value {!r} for field {} is not of type {}".format( |
| self.__class__.__name__, default, name, types_str)) |
| self.default = default |
| |
| if c_default: |
| self.c_default = c_default |
| if py_default: |
| self.py_default = py_default |
| |
| if annotation is not unspecified: |
| fail("The 'annotation' parameter is not currently permitted.") |
| |
| # this is deliberate, to prevent you from caching information |
| # about the function in the init. |
| # (that breaks if we get cloned.) |
| # so after this change we will noisily fail. |
| self.function = LandMine("Don't access members of self.function inside converter_init!") |
| self.converter_init(**kwargs) |
| self.function = function |
| |
| def converter_init(self): |
| pass |
| |
| def is_optional(self) -> bool: |
| return (self.default is not unspecified) |
| |
| def _render_self(self, parameter: str, data: CRenderData) -> None: |
| self.parameter = parameter |
| name = self.parser_name |
| |
| # impl_arguments |
| s = ("&" if self.impl_by_reference else "") + name |
| data.impl_arguments.append(s) |
| if self.length: |
| data.impl_arguments.append(self.length_name()) |
| |
| # impl_parameters |
| data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) |
| if self.length: |
| data.impl_parameters.append("Py_ssize_t " + self.length_name()) |
| |
| def _render_non_self(self, parameter, data): |
| self.parameter = parameter |
| name = self.name |
| |
| # declarations |
| d = self.declaration(in_parser=True) |
| data.declarations.append(d) |
| |
| # initializers |
| initializers = self.initialize() |
| if initializers: |
| data.initializers.append('/* initializers for ' + name + ' */\n' + initializers.rstrip()) |
| |
| # modifications |
| modifications = self.modify() |
| if modifications: |
| data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip()) |
| |
| # keywords |
| if parameter.is_vararg(): |
| pass |
| elif parameter.is_positional_only(): |
| data.keywords.append('') |
| else: |
| data.keywords.append(parameter.name) |
| |
| # format_units |
| if self.is_optional() and '|' not in data.format_units: |
| data.format_units.append('|') |
| if parameter.is_keyword_only() and '$' not in data.format_units: |
| data.format_units.append('$') |
| data.format_units.append(self.format_unit) |
| |
| # parse_arguments |
| self.parse_argument(data.parse_arguments) |
| |
| # post_parsing |
| if post_parsing := self.post_parsing(): |
| data.post_parsing.append('/* Post parse cleanup for ' + name + ' */\n' + post_parsing.rstrip() + '\n') |
| |
| # cleanup |
| cleanup = self.cleanup() |
| if cleanup: |
| data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n") |
| |
| def render(self, parameter: str, data: CRenderData) -> None: |
| """ |
| parameter is a clinic.Parameter instance. |
| data is a CRenderData instance. |
| """ |
| self._render_self(parameter, data) |
| self._render_non_self(parameter, data) |
| |
| def length_name(self): |
| """Computes the name of the associated "length" variable.""" |
| if not self.length: |
| return None |
| return self.parser_name + "_length" |
| |
| # Why is this one broken out separately? |
| # For "positional-only" function parsing, |
| # which generates a bunch of PyArg_ParseTuple calls. |
| def parse_argument(self, list): |
| assert not (self.converter and self.encoding) |
| if self.format_unit == 'O&': |
| assert self.converter |
| list.append(self.converter) |
| |
| if self.encoding: |
| list.append(c_repr(self.encoding)) |
| elif self.subclass_of: |
| list.append(self.subclass_of) |
| |
| s = ("&" if self.parse_by_reference else "") + self.name |
| list.append(s) |
| |
| if self.length: |
| list.append("&" + self.length_name()) |
| |
| # |
| # All the functions after here are intended as extension points. |
| # |
| |
| def simple_declaration(self, by_reference=False, *, in_parser=False): |
| """ |
| Computes the basic declaration of the variable. |
| Used in computing the prototype declaration and the |
| variable declaration. |
| """ |
| prototype = [self.type] |
| if by_reference or not self.type.endswith('*'): |
| prototype.append(" ") |
| if by_reference: |
| prototype.append('*') |
| if in_parser: |
| name = self.parser_name |
| else: |
| name = self.name |
| if self.unused: |
| name = f"Py_UNUSED({name})" |
| prototype.append(name) |
| return "".join(prototype) |
| |
| def declaration(self, *, in_parser=False): |
| """ |
| The C statement to declare this variable. |
| """ |
| declaration = [self.simple_declaration(in_parser=True)] |
| default = self.c_default |
| if not default and self.parameter.group: |
| default = self.c_ignored_default |
| if default: |
| declaration.append(" = ") |
| declaration.append(default) |
| declaration.append(";") |
| if self.length: |
| declaration.append('\nPy_ssize_t ') |
| declaration.append(self.length_name()) |
| declaration.append(';') |
| return "".join(declaration) |
| |
| def initialize(self) -> str: |
| """ |
| The C statements required to set up this variable before parsing. |
| Returns a string containing this code indented at column 0. |
| If no initialization is necessary, returns an empty string. |
| """ |
| return "" |
| |
| def modify(self) -> str: |
| """ |
| The C statements required to modify this variable after parsing. |
| Returns a string containing this code indented at column 0. |
| If no modification is necessary, returns an empty string. |
| """ |
| return "" |
| |
| def post_parsing(self) -> str: |
| """ |
| The C statements required to do some operations after the end of parsing but before cleaning up. |
| Return a string containing this code indented at column 0. |
| If no operation is necessary, return an empty string. |
| """ |
| return "" |
| |
| def cleanup(self) -> str: |
| """ |
| The C statements required to clean up after this variable. |
| Returns a string containing this code indented at column 0. |
| If no cleanup is necessary, returns an empty string. |
| """ |
| return "" |
| |
| def pre_render(self): |
| """ |
| A second initialization function, like converter_init, |
| called just before rendering. |
| You are permitted to examine self.function here. |
| """ |
| pass |
| |
| def parse_arg(self, argname, displayname): |
| if self.format_unit == 'O&': |
| return """ |
| if (!{converter}({argname}, &{paramname})) {{{{ |
| goto exit; |
| }}}} |
| """.format(argname=argname, paramname=self.parser_name, |
| converter=self.converter) |
| if self.format_unit == 'O!': |
| cast = '(%s)' % self.type if self.type != 'PyObject *' else '' |
| if self.subclass_of in type_checks: |
| typecheck, typename = type_checks[self.subclass_of] |
| return """ |
| if (!{typecheck}({argname})) {{{{ |
| _PyArg_BadArgument("{{name}}", {displayname}, "{typename}", {argname}); |
| goto exit; |
| }}}} |
| {paramname} = {cast}{argname}; |
| """.format(argname=argname, paramname=self.parser_name, |
| displayname=displayname, typecheck=typecheck, |
| typename=typename, cast=cast) |
| return """ |
| if (!PyObject_TypeCheck({argname}, {subclass_of})) {{{{ |
| _PyArg_BadArgument("{{name}}", {displayname}, ({subclass_of})->tp_name, {argname}); |
| goto exit; |
| }}}} |
| {paramname} = {cast}{argname}; |
| """.format(argname=argname, paramname=self.parser_name, |
| subclass_of=self.subclass_of, cast=cast, |
| displayname=displayname) |
| if self.format_unit == 'O': |
| cast = '(%s)' % self.type if self.type != 'PyObject *' else '' |
| return """ |
| {paramname} = {cast}{argname}; |
| """.format(argname=argname, paramname=self.parser_name, cast=cast) |
| return None |
| |
| def set_template_dict(self, template_dict: TemplateDict) -> None: |
| pass |
| |
| @property |
| def parser_name(self): |
| if self.name in CLINIC_PREFIXED_ARGS: # bpo-39741 |
| return CLINIC_PREFIX + self.name |
| else: |
| return self.name |
| |
| type_checks = { |
| '&PyLong_Type': ('PyLong_Check', 'int'), |
| '&PyTuple_Type': ('PyTuple_Check', 'tuple'), |
| '&PyList_Type': ('PyList_Check', 'list'), |
| '&PySet_Type': ('PySet_Check', 'set'), |
| '&PyFrozenSet_Type': ('PyFrozenSet_Check', 'frozenset'), |
| '&PyDict_Type': ('PyDict_Check', 'dict'), |
| '&PyUnicode_Type': ('PyUnicode_Check', 'str'), |
| '&PyBytes_Type': ('PyBytes_Check', 'bytes'), |
| '&PyByteArray_Type': ('PyByteArray_Check', 'bytearray'), |
| } |
| |
| |
| ConverterType = Callable[..., CConverter] |
| ConverterDict = dict[str, ConverterType] |
| |
| # maps strings to callables. |
| # these callables must be of the form: |
| # def foo(name, default, *, ...) |
| # The callable may have any number of keyword-only parameters. |
| # The callable must return a CConverter object. |
| # The callable should not call builtins.print. |
| converters: ConverterDict = {} |
| |
| # maps strings to callables. |
| # these callables follow the same rules as those for "converters" above. |
| # note however that they will never be called with keyword-only parameters. |
| legacy_converters: ConverterDict = {} |
| |
| # maps strings to callables. |
| # these callables must be of the form: |
| # def foo(*, ...) |
| # The callable may have any number of keyword-only parameters. |
| # The callable must return a CReturnConverter object. |
| # The callable should not call builtins.print. |
| ReturnConverterDict = dict[str, ReturnConverterType] |
| return_converters: ReturnConverterDict = {} |
| |
| TypeSet = set[bltns.type[Any]] |
| |
| |
| class bool_converter(CConverter): |
| type = 'int' |
| default_type = bool |
| format_unit = 'p' |
| c_ignored_default = '0' |
| |
| def converter_init(self, *, accept: TypeSet = {object}) -> None: |
| if accept == {int}: |
| self.format_unit = 'i' |
| elif accept != {object}: |
| fail("bool_converter: illegal 'accept' argument " + repr(accept)) |
| if self.default is not unspecified: |
| self |