| # |
| # Secret Labs' Regular Expression Engine |
| # |
| # convert template to internal format |
| # |
| # Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. |
| # |
| # See the __init__.py file for information on usage and redistribution. |
| # |
| |
| """Internal support module for sre""" |
| |
| import _sre |
| from . import _parser |
| from ._constants import * |
| from ._casefix import _EXTRA_CASES |
| from ._optimizer import ( |
| _combine_flags, _compile_charset, _optimize_charset, _compile_info, |
| _simple, _CHARSET_ALL, _CODEBITS, MAXCODE, optimize, |
| ) |
| |
| assert _sre.MAGIC == MAGIC, "SRE module mismatch" |
| |
| _LITERAL_CODES = {LITERAL, NOT_LITERAL} |
| _SUCCESS_CODES = {SUCCESS, FAILURE} |
| _ASSERT_CODES = {ASSERT, ASSERT_NOT} |
| |
| _REPEATING_CODES = { |
| MIN_REPEAT: (REPEAT, MIN_UNTIL, MIN_REPEAT_ONE), |
| MAX_REPEAT: (REPEAT, MAX_UNTIL, REPEAT_ONE), |
| POSSESSIVE_REPEAT: (POSSESSIVE_REPEAT, SUCCESS, POSSESSIVE_REPEAT_ONE), |
| } |
| |
| def _compile(code, pattern, flags): |
| # internal: compile a (sub)pattern |
| emit = code.append |
| _len = len |
| LITERAL_CODES = _LITERAL_CODES |
| REPEATING_CODES = _REPEATING_CODES |
| SUCCESS_CODES = _SUCCESS_CODES |
| ASSERT_CODES = _ASSERT_CODES |
| iscased = None |
| tolower = None |
| fixes = None |
| if flags & SRE_FLAG_IGNORECASE and not flags & SRE_FLAG_LOCALE: |
| if flags & SRE_FLAG_UNICODE: |
| iscased = _sre.unicode_iscased |
| tolower = _sre.unicode_tolower |
| fixes = _EXTRA_CASES |
| else: |
| iscased = _sre.ascii_iscased |
| tolower = _sre.ascii_tolower |
| for op, av in pattern: |
| if op in LITERAL_CODES: |
| if not flags & SRE_FLAG_IGNORECASE: |
| emit(op) |
| emit(av) |
| elif flags & SRE_FLAG_LOCALE: |
| emit(OP_LOCALE_IGNORE[op]) |
| emit(av) |
| elif not iscased(av): |
| emit(op) |
| emit(av) |
| else: |
| lo = tolower(av) |
| if not fixes: # ascii |
| emit(OP_IGNORE[op]) |
| emit(lo) |
| elif lo not in fixes: |
| emit(OP_UNICODE_IGNORE[op]) |
| emit(lo) |
| else: |
| emit(IN_UNI_IGNORE) |
| skip = _len(code); emit(0) |
| if op is NOT_LITERAL: |
| emit(NEGATE) |
| for k in (lo,) + fixes[lo]: |
| emit(LITERAL) |
| emit(k) |
| emit(FAILURE) |
| code[skip] = _len(code) - skip |
| elif op is IN: |
| charset, hascased = _optimize_charset(av, iscased, tolower, fixes) |
| if not charset: |
| emit(FAILURE) |
| elif charset == _CHARSET_ALL: |
| emit(ANY_ALL) |
| else: |
| if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: |
| emit(IN_LOC_IGNORE) |
| elif not hascased: |
| emit(IN) |
| elif not fixes: # ascii |
| emit(IN_IGNORE) |
| else: |
| emit(IN_UNI_IGNORE) |
| skip = _len(code); emit(0) |
| _compile_charset(charset, flags, code) |
| code[skip] = _len(code) - skip |
| elif op is ANY: |
| if flags & SRE_FLAG_DOTALL: |
| emit(ANY_ALL) |
| else: |
| emit(ANY) |
| elif op in REPEATING_CODES: |
| if _simple(av[2]): |
| emit(REPEATING_CODES[op][2]) |
| skip = _len(code); emit(0) |
| emit(av[0]) |
| emit(av[1]) |
| _compile(code, av[2], flags) |
| emit(SUCCESS) |
| code[skip] = _len(code) - skip |
| else: |
| emit(REPEATING_CODES[op][0]) |
| skip = _len(code); emit(0) |
| emit(av[0]) |
| emit(av[1]) |
| _compile(code, av[2], flags) |
| code[skip] = _len(code) - skip |
| emit(REPEATING_CODES[op][1]) |
| elif op is SUBPATTERN: |
| group, add_flags, del_flags, p = av |
| if group: |
| emit(MARK) |
| emit((group-1)*2) |
| # _compile_info(code, p, _combine_flags(flags, add_flags, del_flags)) |
| _compile(code, p, _combine_flags(flags, add_flags, del_flags)) |
| if group: |
| emit(MARK) |
| emit((group-1)*2+1) |
| elif op is ATOMIC_GROUP: |
| # Atomic Groups are handled by starting with an Atomic |
| # Group op code, then putting in the atomic group pattern |
| # and finally a success op code to tell any repeat |
| # operations within the Atomic Group to stop eating and |
| # pop their stack if they reach it |
| emit(ATOMIC_GROUP) |
| skip = _len(code); emit(0) |
| _compile(code, av, flags) |
| emit(SUCCESS) |
| code[skip] = _len(code) - skip |
| elif op in SUCCESS_CODES: |
| emit(op) |
| elif op in ASSERT_CODES: |
| emit(op) |
| skip = _len(code); emit(0) |
| if av[0] >= 0: |
| emit(0) # look ahead |
| else: |
| lo, hi = av[1].getwidth() |
| if lo > MAXCODE: |
| raise error("looks too much behind") |
| if lo != hi: |
| raise PatternError("look-behind requires fixed-width pattern") |
| emit(lo) # look behind |
| _compile(code, av[1], flags) |
| emit(SUCCESS) |
| code[skip] = _len(code) - skip |
| elif op is AT: |
| emit(op) |
| if flags & SRE_FLAG_MULTILINE: |
| av = AT_MULTILINE.get(av, av) |
| if flags & SRE_FLAG_LOCALE: |
| av = AT_LOCALE.get(av, av) |
| elif flags & SRE_FLAG_UNICODE: |
| av = AT_UNICODE.get(av, av) |
| emit(av) |
| elif op is BRANCH: |
| emit(op) |
| tail = [] |
| tailappend = tail.append |
| for av in av[1]: |
| skip = _len(code); emit(0) |
| # _compile_info(code, av, flags) |
| _compile(code, av, flags) |
| emit(JUMP) |
| tailappend(_len(code)); emit(0) |
| code[skip] = _len(code) - skip |
| emit(FAILURE) # end of branch |
| for tail in tail: |
| code[tail] = _len(code) - tail |
| elif op is CATEGORY: |
| emit(op) |
| if flags & SRE_FLAG_LOCALE: |
| av = CH_LOCALE[av] |
| elif flags & SRE_FLAG_UNICODE: |
| av = CH_UNICODE[av] |
| emit(av) |
| elif op is GROUPREF: |
| if not flags & SRE_FLAG_IGNORECASE: |
| emit(op) |
| elif flags & SRE_FLAG_LOCALE: |
| emit(GROUPREF_LOC_IGNORE) |
| elif not fixes: # ascii |
| emit(GROUPREF_IGNORE) |
| else: |
| emit(GROUPREF_UNI_IGNORE) |
| emit(av-1) |
| elif op is GROUPREF_EXISTS: |
| emit(op) |
| emit(av[0]-1) |
| skipyes = _len(code); emit(0) |
| _compile(code, av[1], flags) |
| if av[2]: |
| emit(JUMP) |
| skipno = _len(code); emit(0) |
| code[skipyes] = _len(code) - skipyes + 1 |
| _compile(code, av[2], flags) |
| code[skipno] = _len(code) - skipno |
| else: |
| code[skipyes] = _len(code) - skipyes + 1 |
| else: |
| raise PatternError(f"internal: unsupported operand type {op!r}") |
| |
| def isstring(obj): |
| return isinstance(obj, (str, bytes)) |
| |
| def _code(p, flags): |
| |
| flags = p.state.flags | flags |
| |
| # run the optimizer passes over the parsed pattern |
| optimize(p) |
| |
| code = [] |
| |
| # compile info block |
| _compile_info(code, p, flags) |
| |
| # compile the pattern |
| _compile(code, p.data, flags) |
| |
| code.append(SUCCESS) |
| |
| return code |
| |
| def _hex_code(code): |
| return '[%s]' % ', '.join('%#0*x' % (_sre.CODESIZE*2+2, x) for x in code) |
| |
| def dis(code): |
| import sys |
| |
| labels = set() |
| level = 0 |
| offset_width = len(str(len(code) - 1)) |
| |
| def dis_(start, end): |
| def print_(*args, to=None): |
| if to is not None: |
| labels.add(to) |
| args += ('(to %d)' % (to,),) |
| print('%*d%s ' % (offset_width, start, ':' if start in labels else '.'), |
| end=' '*(level-1)) |
| print(*args) |
| |
| def print_2(*args): |
| print(end=' '*(offset_width + 2*level)) |
| print(*args) |
| |
| nonlocal level |
| level += 1 |
| i = start |
| while i < end: |
| start = i |
| op = code[i] |
| i += 1 |
| op = OPCODES[op] |
| if op in (SUCCESS, FAILURE, ANY, ANY_ALL, |
| MAX_UNTIL, MIN_UNTIL, NEGATE): |
| print_(op) |
| elif op in (LITERAL, NOT_LITERAL, |
| LITERAL_IGNORE, NOT_LITERAL_IGNORE, |
| LITERAL_UNI_IGNORE, NOT_LITERAL_UNI_IGNORE, |
| LITERAL_LOC_IGNORE, NOT_LITERAL_LOC_IGNORE): |
| arg = code[i] |
| i += 1 |
| print_(op, '%#02x (%r)' % (arg, chr(arg))) |
| elif op is AT: |
| arg = code[i] |
| i += 1 |
| arg = str(ATCODES[arg]) |
| assert arg[:3] == 'AT_' |
| print_(op, arg[3:]) |
| elif op is CATEGORY: |
| arg = code[i] |
| i += 1 |
| arg = str(CHCODES[arg]) |
| assert arg[:9] == 'CATEGORY_' |
| print_(op, arg[9:]) |
| elif op in (IN, IN_IGNORE, IN_UNI_IGNORE, IN_LOC_IGNORE): |
| skip = code[i] |
| print_(op, skip, to=i+skip) |
| dis_(i+1, i+skip) |
| i += skip |
| elif op in (RANGE, RANGE_UNI_IGNORE): |
| lo, hi = code[i: i+2] |
| i += 2 |
| print_(op, '%#02x %#02x (%r-%r)' % (lo, hi, chr(lo), chr(hi))) |
| elif op is CHARSET: |
| print_(op, _hex_code(code[i: i + 256//_CODEBITS])) |
| i += 256//_CODEBITS |
| elif op is BIGCHARSET: |
| arg = code[i] |
| i += 1 |
| mapping = list(b''.join(x.to_bytes(_sre.CODESIZE, sys.byteorder) |
| for x in code[i: i + 256//_sre.CODESIZE])) |
| print_(op, arg, mapping) |
| i += 256//_sre.CODESIZE |
| level += 1 |
| for j in range(arg): |
| print_2(_hex_code(code[i: i + 256//_CODEBITS])) |
| i += 256//_CODEBITS |
| level -= 1 |
| elif op in (MARK, GROUPREF, GROUPREF_IGNORE, GROUPREF_UNI_IGNORE, |
| GROUPREF_LOC_IGNORE): |
| arg = code[i] |
| i += 1 |
| print_(op, arg) |
| elif op is JUMP: |
| skip = code[i] |
| print_(op, skip, to=i+skip) |
| i += 1 |
| elif op is BRANCH: |
| skip = code[i] |
| print_(op, skip, to=i+skip) |
| while skip: |
| dis_(i+1, i+skip) |
| i += skip |
| start = i |
| skip = code[i] |
| if skip: |
| print_('branch', skip, to=i+skip) |
| else: |
| print_(FAILURE) |
| i += 1 |
| elif op in (REPEAT, REPEAT_ONE, MIN_REPEAT_ONE, |
| POSSESSIVE_REPEAT, POSSESSIVE_REPEAT_ONE): |
| skip, min, max = code[i: i+3] |
| if max == MAXREPEAT: |
| max = 'MAXREPEAT' |
| print_(op, skip, min, max, to=i+skip) |
| dis_(i+3, i+skip) |
| i += skip |
| elif op is GROUPREF_EXISTS: |
| arg, skip = code[i: i+2] |
| print_(op, arg, skip, to=i+skip) |
| i += 2 |
| elif op in (ASSERT, ASSERT_NOT): |
| skip, arg = code[i: i+2] |
| print_(op, skip, arg, to=i+skip) |
| dis_(i+2, i+skip) |
| i += skip |
| elif op is ATOMIC_GROUP: |
| skip = code[i] |
| print_(op, skip, to=i+skip) |
| dis_(i+1, i+skip) |
| i += skip |
| elif op is INFO: |
| skip, flags, min, max = code[i: i+4] |
| if max == MAXREPEAT: |
| max = 'MAXREPEAT' |
| print_(op, skip, bin(flags), min, max, to=i+skip) |
| start = i+4 |
| if flags & SRE_INFO_PREFIX: |
| prefix_len, prefix_skip = code[i+4: i+6] |
| print_2(' prefix_skip', prefix_skip) |
| start = i + 6 |
| prefix = code[start: start+prefix_len] |
| print_2(' prefix', |
| '[%s]' % ', '.join('%#02x' % x for x in prefix), |
| '(%r)' % ''.join(map(chr, prefix))) |
| start += prefix_len |
| print_2(' overlap', code[start: start+prefix_len]) |
| start += prefix_len |
| if flags & SRE_INFO_CHARSET: |
| level += 1 |
| print_2('in') |
| dis_(start, i+skip) |
| level -= 1 |
| i += skip |
| else: |
| raise ValueError(op) |
| |
| level -= 1 |
| |
| dis_(0, len(code)) |
| |
| |
| def compile(p, flags=0): |
| # internal: convert pattern list to internal format |
| |
| if isstring(p): |
| pattern = p |
| p = _parser.parse(p, flags) |
| else: |
| pattern = None |
| |
| code = _code(p, flags) |
| |
| if flags & SRE_FLAG_DEBUG: |
| print() |
| dis(code) |
| |
| # map in either direction |
| groupindex = p.state.groupdict |
| indexgroup = [None] * p.state.groups |
| for k, i in groupindex.items(): |
| indexgroup[i] = k |
| |
| return _sre.compile( |
| pattern, flags | p.state.flags, code, |
| p.state.groups-1, |
| groupindex, tuple(indexgroup) |
| ) |
