| import os.path |
| import re |
| |
| from . import common as _common |
| |
| # The following C files must not built with Py_BUILD_CORE. |
| FILES_WITHOUT_INTERNAL_CAPI = frozenset(( |
| # Modules/ |
| '_testcapimodule.c', |
| '_testlimitedcapi.c', |
| '_testclinic_limited.c', |
| 'xxlimited.c', |
| 'xxlimited_35.c', |
| )) |
| |
| # C files in the fhe following directories must not be built with |
| # Py_BUILD_CORE. |
| DIRS_WITHOUT_INTERNAL_CAPI = frozenset(( |
| '_testcapi', # Modules/_testcapi/ |
| '_testlimitedcapi', # Modules/_testlimitedcapi/ |
| )) |
| |
| TOOL = 'gcc' |
| |
| META_FILES = { |
| '<built-in>', |
| '<command-line>', |
| } |
| |
| # https://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html |
| # flags: |
| # 1 start of a new file |
| # 2 returning to a file (after including another) |
| # 3 following text comes from a system header file |
| # 4 following text treated wrapped in implicit extern "C" block |
| LINE_MARKER_RE = re.compile(r'^# (\d+) "([^"]+)"((?: [1234])*)$') |
| PREPROC_DIRECTIVE_RE = re.compile(r'^\s*#\s*(\w+)\b.*') |
| COMPILER_DIRECTIVE_RE = re.compile(r''' |
| ^ |
| (.*?) # <before> |
| (__\w+__) # <directive> |
| \s* |
| [(] [(] |
| ( |
| [^()]* |
| (?: |
| [(] |
| [^()]* |
| [)] |
| [^()]* |
| )* |
| ) # <args> |
| ( [)] [)] ) # <closed> |
| ''', re.VERBOSE) |
| |
| POST_ARGS = ( |
| '-pthread', |
| '-std=c99', |
| #'-g', |
| #'-Og', |
| #'-Wno-unused-result', |
| #'-Wsign-compare', |
| #'-Wall', |
| #'-Wextra', |
| '-E', |
| ) |
| |
| EXIT_MARKERS = {'# 0 "<command-line>" 2', '# 1 "<command-line>" 2'} |
| |
| |
| def preprocess(filename, |
| incldirs=None, |
| includes=None, |
| macros=None, |
| samefiles=None, |
| cwd=None, |
| ): |
| if not cwd or not os.path.isabs(cwd): |
| cwd = os.path.abspath(cwd or '.') |
| filename = _normpath(filename, cwd) |
| |
| postargs = POST_ARGS |
| basename = os.path.basename(filename) |
| dirname = os.path.basename(os.path.dirname(filename)) |
| if (basename not in FILES_WITHOUT_INTERNAL_CAPI |
| and dirname not in DIRS_WITHOUT_INTERNAL_CAPI): |
| postargs += ('-DPy_BUILD_CORE=1',) |
| |
| text = _common.preprocess( |
| TOOL, |
| filename, |
| incldirs=incldirs, |
| includes=includes, |
| macros=macros, |
| #preargs=PRE_ARGS, |
| postargs=postargs, |
| executable=['gcc'], |
| compiler='unix', |
| cwd=cwd, |
| ) |
| return _iter_lines(text, filename, samefiles, cwd) |
| |
| |
| def _iter_lines(text, reqfile, samefiles, cwd, raw=False): |
| lines = iter(text.splitlines()) |
| |
| # The first line is special. |
| # The next two lines are consistent. |
| firstlines = [ |
| f'# 0 "{reqfile}"', |
| '# 0 "<built-in>"', |
| '# 0 "<command-line>"', |
| ] |
| if text.startswith('# 1 '): |
| # Some preprocessors emit a lineno of 1 for line-less entries. |
| firstlines = [l.replace('# 0 ', '# 1 ') for l in firstlines] |
| for expected in firstlines: |
| line = next(lines) |
| if line != expected: |
| raise NotImplementedError((line, expected)) |
| |
| # Do all the CLI-provided includes. |
| filter_reqfile = (lambda f: _filter_reqfile(f, reqfile, samefiles)) |
| make_info = (lambda lno: _common.FileInfo(reqfile, lno)) |
| last = None |
| for line in lines: |
| assert last != reqfile, (last,) |
| lno, included, flags = _parse_marker_line(line, reqfile) |
| if not included: |
| raise NotImplementedError((line,)) |
| if included == reqfile: |
| # This will be the last one. |
| assert not flags, (line, flags) |
| else: |
| assert 1 in flags, (line, flags) |
| yield from _iter_top_include_lines( |
| lines, |
| _normpath(included, cwd), |
| cwd, |
| filter_reqfile, |
| make_info, |
| raw, |
| EXIT_MARKERS |
| ) |
| last = included |
| # The last one is always the requested file. |
| assert included == reqfile, (line,) |
| |
| |
| def _iter_top_include_lines(lines, topfile, cwd, |
| filter_reqfile, make_info, |
| raw, exit_markers): |
| partial = 0 # depth |
| files = [topfile] |
| # We start at 1 in case there are source lines (including blank ones) |
| # before the first marker line. Also, we already verified in |
| # _parse_marker_line() that the preprocessor reported lno as 1. |
| lno = 1 |
| for line in lines: |
| if line in exit_markers: |
| # We're done with this top-level include. |
| return |
| |
| _lno, included, flags = _parse_marker_line(line) |
| if included: |
| # HACK: |
| # Mixes curses.h and ncurses.h marker lines |
| # gcc silently passes this, while clang fails |
| # See: /Include/py_curses.h #if-elif directives |
| # And compare with preprocessor output |
| if os.path.basename(included) == 'curses.h': |
| included = os.path.join(os.path.dirname(included), 'ncurses.h') |
| |
| lno = _lno |
| included = _normpath(included, cwd) |
| # We hit a marker line. |
| if 1 in flags: |
| # We're entering a file. |
| # XXX Cycles are unexpected? |
| #assert included not in files, (line, files) |
| files.append(included) |
| elif 2 in flags: |
| # We're returning to a file. |
| assert files and included in files, (line, files) |
| assert included != files[-1], (line, files) |
| while files[-1] != included: |
| files.pop() |
| # XXX How can a file return to line 1? |
| #assert lno > 1, (line, lno) |
| else: |
| if included == files[-1]: |
| # It's the next line from the file. |
| assert lno > 1, (line, lno) |
| else: |
| # We ran into a user-added #LINE directive, |
| # which we promptly ignore. |
| pass |
| elif not files: |
| raise NotImplementedError((line,)) |
| elif filter_reqfile(files[-1]): |
| assert lno is not None, (line, files[-1]) |
| if (m := PREPROC_DIRECTIVE_RE.match(line)): |
| name, = m.groups() |
| if name != 'pragma': |
| raise Exception(line) |
| else: |
| line = re.sub(r'__inline__', 'inline', line) |
| if not raw: |
| line, partial = _strip_directives(line, partial=partial) |
| yield _common.SourceLine( |
| make_info(lno), |
| 'source', |
| line or '', |
| None, |
| ) |
| lno += 1 |
| |
| |
| def _parse_marker_line(line, reqfile=None): |
| m = LINE_MARKER_RE.match(line) |
| if not m: |
| return None, None, None |
| lno, origfile, flags = m.groups() |
| lno = int(lno) |
| assert origfile not in META_FILES, (line,) |
| assert lno > 0, (line, lno) |
| flags = set(int(f) for f in flags.split()) if flags else () |
| |
| if 1 in flags: |
| # We're entering a file. |
| assert lno == 1, (line, lno) |
| assert 2 not in flags, (line,) |
| elif 2 in flags: |
| # We're returning to a file. |
| #assert lno > 1, (line, lno) |
| pass |
| elif reqfile and origfile == reqfile: |
| # We're starting the requested file. |
| assert lno == 1, (line, lno) |
| assert not flags, (line, flags) |
| else: |
| # It's the next line from the file. |
| assert lno > 1, (line, lno) |
| return lno, origfile, flags |
| |
| |
| def _strip_directives(line, partial=0): |
| # We assume there are no string literals with parens in directive bodies. |
| while partial > 0: |
| if not (m := re.match(r'[^{}]*([()])', line)): |
| return None, partial |
| delim, = m.groups() |
| partial += 1 if delim == '(' else -1 # opened/closed |
| line = line[m.end():] |
| |
| line = re.sub(r'__extension__', '', line) |
| line = re.sub(r'__thread\b', '_Thread_local', line) |
| |
| while (m := COMPILER_DIRECTIVE_RE.match(line)): |
| before, _, _, closed = m.groups() |
| if closed: |
| line = f'{before} {line[m.end():]}' |
| else: |
| after, partial = _strip_directives(line[m.end():], 2) |
| line = f'{before} {after or ""}' |
| if partial: |
| break |
| |
| return line, partial |
| |
| |
| def _filter_reqfile(current, reqfile, samefiles): |
| if current == reqfile: |
| return True |
| if current == '<stdin>': |
| return True |
| if current in samefiles: |
| return True |
| return False |
| |
| |
| def _normpath(filename, cwd): |
| assert cwd |
| return os.path.normpath(os.path.join(cwd, filename)) |
