| #!/usr/bin/python | |
| """Generate type/module slot files | |
| """ | |
| # See the input file (Python/slots.toml) for a description of its format. | |
| import io | |
| import sys | |
| import json | |
| import tomllib | |
| import argparse | |
| import functools | |
| import contextlib | |
| import collections | |
| from pathlib import Path | |
| GENERATED_BY = 'Generated by Tools/build/generate_slots.py' | |
| REPO_ROOT = Path(__file__).parent.parent.parent | |
| DEFAULT_INPUT_PATH = REPO_ROOT / 'Python/slots.toml' | |
| INCLUDE_PATH = REPO_ROOT / 'Include' | |
| DEFAULT_PUBLIC_HEADER_PATH = INCLUDE_PATH / 'slots_generated.h' | |
| DEFAULT_PRIVATE_HEADER_PATH = INCLUDE_PATH / 'internal/pycore_slots_generated.h' | |
| DEFAULT_C_PATH = REPO_ROOT / 'Python/slots_generated.c' | |
| TABLES = { | |
| 'tp': 'ht_type', | |
| 'am': 'as_async', | |
| 'nb': 'as_number', | |
| 'mp': 'as_mapping', | |
| 'sq': 'as_sequence', | |
| 'bf': 'as_buffer', | |
| } | |
| class SlotInfo: | |
| def __init__(self, id, data): | |
| self.id = id | |
| self.kind = data['kind'] | |
| self._data = data | |
| try: | |
| self.name = data['name'] | |
| except KeyError: | |
| self.name = '/'.join(data["equivalents"].values()) | |
| else: | |
| assert self.name.isidentifier | |
| @functools.cached_property | |
| def equivalents(self): | |
| return self._data['equivalents'] | |
| @functools.cached_property | |
| def dtype(self): | |
| try: | |
| return self._data['dtype'] | |
| except KeyError: | |
| if self.is_type_field: | |
| return 'func' | |
| raise | |
| @functools.cached_property | |
| def functype(self): | |
| return self._data['functype'] | |
| @functools.cached_property | |
| def is_type_field(self): | |
| return self._data.get('is_type_field') | |
| @functools.cached_property | |
| def type_field(self): | |
| assert self.is_type_field | |
| return self._data.get('field', self.name.removeprefix('Py_')) | |
| @functools.cached_property | |
| def type_table_ident(self): | |
| assert self.is_type_field | |
| return self._data.get('table', self.type_field[:2]) | |
| @functools.cached_property | |
| def duplicate_handling(self): | |
| return self._data.get('duplicates', 'reject') | |
| @functools.cached_property | |
| def null_handling(self): | |
| try: | |
| return self._data['nulls'] | |
| except KeyError: | |
| if self.kind == 'compat': | |
| return 'allow' | |
| if self.dtype in {'ptr', 'func'}: | |
| return 'reject' | |
| return 'allow' | |
| @functools.cached_property | |
| def must_be_static(self): | |
| return self._data.get('must_be_static', False) | |
| def parse_slots(file): | |
| toml_contents = tomllib.load(file) | |
| result = [None] * len(toml_contents) | |
| for key, data in toml_contents.items(): | |
| slot_id = int(key) | |
| try: | |
| if result[slot_id]: | |
| raise ValueError(f'slot ID {slot_id} repeated') | |
| result[slot_id] = SlotInfo(slot_id, data) | |
| except Exception as e: | |
| e.add_note(f'handling slot {slot_id}') | |
| raise | |
| return result | |
| class CWriter: | |
| """Simple helper for generating C code""" | |
| def __init__(self, file): | |
| self.file = file | |
| self.indent = '' | |
| self(f'/* {GENERATED_BY} */') | |
| self() | |
| def out(self, *args, **kwargs): | |
| """print args to the file, with current indent at the start""" | |
| print(self.indent, end='', file=self.file) | |
| print(*args, file=self.file, **kwargs) | |
| __call__ = out | |
| @contextlib.contextmanager | |
| def block(self, header=None, end=''): | |
| """Context for a {}-enclosed block of C""" | |
| if header is None: | |
| self.out('{') | |
| else: | |
| self.out(header, '{') | |
| old_indent = self.indent | |
| self.indent += ' ' | |
| yield | |
| self.indent = old_indent | |
| self.out('}' + end) | |
| def write_public_header(f, slots): | |
| out = CWriter(f) | |
| out(f'#ifndef _PY_HAVE_SLOTS_GENERATED_H') | |
| out(f'#define _PY_HAVE_SLOTS_GENERATED_H') | |
| out() | |
| out(f'#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15)') | |
| out(f'#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) NEW') | |
| out(f'#else') | |
| out(f'#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) OLD') | |
| out(f'#endif') | |
| out() | |
| compat_ids = {} | |
| for slot in slots: | |
| if slot.kind == 'compat': | |
| for new_name in slot.equivalents.values(): | |
| compat_ids[new_name] = slot.id | |
| for slot in slots: | |
| if slot.kind == 'compat': | |
| continue | |
| slot_id = slot.id | |
| if compat := compat_ids.get(slot.name): | |
| slot_id = f'_Py_SLOT_COMPAT_VALUE({compat}, {slot_id})' | |
| out(f'#define {slot.name} {slot_id}') | |
| out() | |
| out(f'#define _Py_slot_COUNT {len(slots)}') | |
| out(f'#endif /* _PY_HAVE_SLOTS_GENERATED_H */') | |
| def write_private_header(f, slots): | |
| out = CWriter(f) | |
| def add_case(slot): | |
| out(out(f' case {slot.id}:')) | |
| slots_by_name = {slot.name: slot for slot in slots} | |
| out(f'#ifndef _PY_HAVE_INTERNAL_SLOTS_GENERATED_H') | |
| out(f'#define _PY_HAVE_INTERNAL_SLOTS_GENERATED_H') | |
| for kind in 'type', 'mod': | |
| out() | |
| out(f'static inline uint16_t') | |
| out(f'_PySlot_resolve_{kind}_slot(uint16_t slot_id)') | |
| with out.block(): | |
| with out.block('switch (slot_id)'): | |
| good_slots = [] | |
| for slot in slots: | |
| if slot.kind == 'compat': | |
| new_slot = slots_by_name[slot.equivalents[kind]] | |
| out(f'case {slot.id}:') | |
| out(f' return {new_slot.name};') | |
| elif slot.kind in {kind, 'slot'}: | |
| good_slots.append(f'case {slot.name}:') | |
| for case in good_slots: | |
| out(case) | |
| out(f' return slot_id;') | |
| out(f'default:') | |
| out(f' return Py_slot_invalid;') | |
| out() | |
| out(f'static inline void*') | |
| out(f'_PySlot_type_getslot(PyTypeObject *tp, uint16_t slot_id)') | |
| with out.block(): | |
| with out.block('switch (slot_id)'): | |
| for slot in slots: | |
| if slot.is_type_field: | |
| field = slot.type_field | |
| table_ident = slot.type_table_ident | |
| if table_ident == 'tp': | |
| out(f'case {slot.name}:') | |
| out(f' return (void*)tp->{field};') | |
| else: | |
| if table_ident == 'ht': | |
| cond = 'tp->tp_flags & Py_TPFLAGS_HEAPTYPE' | |
| val = f'((PyHeapTypeObject*)tp)->{field}' | |
| else: | |
| table = TABLES[table_ident] | |
| cond = f'tp->tp_{table}' | |
| val = f'tp->tp_{table}->{field}' | |
| out(f'case {slot.name}:') | |
| out(f' if (!({cond})) return NULL;') | |
| out(f' return (void*){val};') | |
| out(f'_PySlot_err_bad_slot("PyType_GetSlot", slot_id);') | |
| out(f'return NULL;') | |
| out() | |
| out(f'static inline void') | |
| out(f'_PySlot_heaptype_apply_field_slot(PyHeapTypeObject *ht,', | |
| f'PySlot slot)') | |
| with out.block(): | |
| with out.block('switch (slot.sl_id)'): | |
| for slot in slots: | |
| if slot.is_type_field: | |
| field = slot.type_field | |
| table_ident = slot.type_table_ident | |
| if table_ident == 'ht': | |
| continue | |
| table = TABLES[table_ident] | |
| if slot.dtype == 'func': | |
| functype = f'({slot.functype})' | |
| else: | |
| functype = '' | |
| out(f'case {slot.name}:') | |
| out(f' ht->{table}.{field} = {functype}slot.sl_{slot.dtype};') | |
| out(f' break;') | |
| out() | |
| out(f'static inline _PySlot_DTYPE') | |
| out(f'_PySlot_get_dtype(uint16_t slot_id)') | |
| with out.block(): | |
| with out.block('switch (slot_id)'): | |
| for slot in slots: | |
| if slot.kind == 'compat': | |
| continue | |
| dtype = slot.dtype | |
| name = slot.name | |
| out(f'case {name}: return _PySlot_DTYPE_{dtype.upper()};') | |
| out(f'default: return _PySlot_DTYPE_VOID;') | |
| out() | |
| out(f'static inline _PySlot_PROBLEM_HANDLING') | |
| out(f'_PySlot_get_duplicate_handling(uint16_t slot_id)') | |
| with out.block(): | |
| with out.block('switch (slot_id)'): | |
| results = collections.defaultdict(list) | |
| for slot in slots: | |
| if slot.kind == 'compat': | |
| continue | |
| handling = slot.duplicate_handling | |
| results[handling.upper()].append(f'case {slot.name}:') | |
| results.pop('REJECT') | |
| for handling, cases in results.items(): | |
| for case in cases: | |
| out(case) | |
| out(f' return _PySlot_PROBLEM_{handling};') | |
| out(f'default:') | |
| out(f' return _PySlot_PROBLEM_REJECT;') | |
| out() | |
| out(f'static inline _PySlot_PROBLEM_HANDLING') | |
| out(f'_PySlot_get_null_handling(uint16_t slot_id)') | |
| with out.block(): | |
| with out.block('switch (slot_id)'): | |
| results = collections.defaultdict(list) | |
| for slot in slots: | |
| if slot.kind == 'compat': | |
| continue | |
| handling = slot.null_handling | |
| if handling is None: | |
| if slot.kind != 'compat' and slot.dtype in {'ptr', 'func'}: | |
| handling = 'reject' | |
| else: | |
| handling = 'allow' | |
| results[handling.upper()].append(f'case {slot.name}:') | |
| results.pop('REJECT') | |
| for handling, cases in results.items(): | |
| for case in cases: | |
| out(case) | |
| out(f' return _PySlot_PROBLEM_{handling};') | |
| out(f'default:') | |
| out(f' return _PySlot_PROBLEM_REJECT;') | |
| out() | |
| out(f'static inline bool') | |
| out(f'_PySlot_get_must_be_static(uint16_t slot_id)') | |
| with out.block(): | |
| with out.block('switch (slot_id)'): | |
| cases = [] | |
| for slot in slots: | |
| if slot.must_be_static: | |
| out(f'case {slot.name}: return true;') | |
| out(f'return false;') | |
| out() | |
| out(f'#endif /* _PY_HAVE_INTERNAL_SLOTS_GENERATED_H */') | |
| def write_c(f, slots): | |
| out = CWriter(f) | |
| out('#include "Python.h"') | |
| out('#include "pycore_slots.h" // _PySlot_names') | |
| out() | |
| with out.block(f'const char *const _PySlot_names[] =', end=';'): | |
| for slot in slots: | |
| out(f'"{slot.name}",') | |
| out('NULL') | |
| @contextlib.contextmanager | |
| def replace_file(filename): | |
| file_path = Path(filename) | |
| with io.StringIO() as sio: | |
| yield sio | |
| try: | |
| old_text = file_path.read_text() | |
| except FileNotFoundError: | |
| old_text = None | |
| new_text = sio.getvalue() | |
| if old_text == new_text: | |
| print(f'{filename}: not modified', file=sys.stderr) | |
| else: | |
| print(f'{filename}: writing new content', file=sys.stderr) | |
| file_path.write_text(new_text) | |
| def main(argv): | |
| if len(argv) == 1: | |
| # No sens calling this with no arguments. | |
| argv.append('--help') | |
| parser = argparse.ArgumentParser(prog=argv[0], description=__doc__) | |
| parser.add_argument( | |
| '-i', '--input', default=DEFAULT_INPUT_PATH, | |
| help=f'the input file (default: {DEFAULT_INPUT_PATH})') | |
| parser.add_argument( | |
| '--generate-all', action=argparse.BooleanOptionalAction, | |
| help='write all output files to their default locations') | |
| parser.add_argument( | |
| '-j', '--jsonl', action=argparse.BooleanOptionalAction, | |
| help='write info to stdout in "JSON Lines" format (one JSON per line)') | |
| outfile_group = parser.add_argument_group( | |
| 'output files', | |
| description='By default, no files are generated. Use --generate-all ' | |
| + 'or the options below to generate them.') | |
| outfile_group.add_argument( | |
| '-H', '--public-header', | |
| help='file into which to write the public header') | |
| outfile_group.add_argument( | |
| '-I', '--private-header', | |
| help='file into which to write the private header') | |
| outfile_group.add_argument( | |
| '-C', '--cfile', | |
| help='file into which to write internal C code') | |
| args = parser.parse_args(argv[1:]) | |
| if args.generate_all: | |
| if args.public_header is None: | |
| args.public_header = DEFAULT_PUBLIC_HEADER_PATH | |
| if args.private_header is None: | |
| args.private_header = DEFAULT_PRIVATE_HEADER_PATH | |
| if args.cfile is None: | |
| args.cfile = DEFAULT_C_PATH | |
| with open(args.input, 'rb') as f: | |
| slots = parse_slots(f) | |
| if args.jsonl: | |
| for slot in slots: | |
| print(json.dumps(slot.to_dict())) | |
| if args.public_header: | |
| with replace_file(args.public_header) as f: | |
| write_public_header(f, slots) | |
| if args.private_header: | |
| with replace_file(args.private_header) as f: | |
| write_private_header(f, slots) | |
| if args.cfile: | |
| with replace_file(args.cfile) as f: | |
| write_c(f, slots) | |
| if __name__ == "__main__": | |
| main(sys.argv) |