Lib/re/_properties.py

#
# Secret Labs' Regular Expression Engine
#
# support for Unicode property escapes \p{...} and \P{...}
#
# See https://unicode.org/reports/tr18/ "Unicode Regular Expressions",
# requirement RL1.2 "Properties".
#
# The supported properties are matched either as CATEGORY opcodes, or as fixed
# sets of character ranges:
#
# * Properties emitted as CATEGORY opcodes (see _CATEGORY_PROPERTIES): \d, \s
#   and \w (as digit, space and word, honouring the ASCII/LOCALE/UNICODE
#   flags), the binary properties Alphabetic, Lowercase, Uppercase, Numeric,
#   Printable, alnum, XID_Start, XID_Continue, Cased and Case_Ignorable, and
#   the POSIX classes blank, graph, print and assigned.
#
# * General_Category values (see _GC_CATEGORY): L, Lt, Nd, Lu, N, Lm, Nl, No,
#   Cf, Z, Zs, C and Cn (combinations of the simple predicates), plus Cc, Cs,
#   Co, Zl and Zp as fixed ranges (see _GC_ANALYTIC).
#
# * Code-point classes given by fixed ranges (see _analytic_ranges): ASCII,
#   Any, Noncharacter_Code_Point, Join_Control, Regional_Indicator, xdigit,
#   ASCII_Hex_Digit, Hex_Digit, cntrl, and the immutable Pattern_Syntax and
#   Pattern_White_Space.
#

from ._constants import (
    IN, CATEGORY, NEGATE, RANGE, LITERAL,
    CATEGORY_DIGIT, CATEGORY_NOT_DIGIT,
    CATEGORY_SPACE, CATEGORY_NOT_SPACE,
    CATEGORY_WORD, CATEGORY_NOT_WORD,
    CATEGORY_ALPHA, CATEGORY_NOT_ALPHA,
    CATEGORY_LOWER, CATEGORY_NOT_LOWER,
    CATEGORY_UPPER, CATEGORY_NOT_UPPER,
    CATEGORY_NUMERIC, CATEGORY_NOT_NUMERIC,
    CATEGORY_PRINTABLE, CATEGORY_NOT_PRINTABLE,
    CATEGORY_ALNUM, CATEGORY_NOT_ALNUM,
    CATEGORY_XID_START, CATEGORY_NOT_XID_START,
    CATEGORY_XID_CONTINUE, CATEGORY_NOT_XID_CONTINUE,
    CATEGORY_TITLE, CATEGORY_NOT_TITLE,
    CATEGORY_CASED, CATEGORY_NOT_CASED,
    CATEGORY_CASE_IGNORABLE, CATEGORY_NOT_CASE_IGNORABLE,
    CATEGORY_LU, CATEGORY_NOT_LU,
    CATEGORY_N, CATEGORY_NOT_N,
    CATEGORY_LM, CATEGORY_NOT_LM,
    CATEGORY_NL, CATEGORY_NOT_NL,
    CATEGORY_NO, CATEGORY_NOT_NO,
    CATEGORY_CF, CATEGORY_NOT_CF,
    CATEGORY_Z, CATEGORY_NOT_Z,
    CATEGORY_ZS, CATEGORY_NOT_ZS,
    CATEGORY_C, CATEGORY_NOT_C,
    CATEGORY_CN, CATEGORY_NOT_CN,
    CATEGORY_ASSIGNED, CATEGORY_NOT_ASSIGNED,
    CATEGORY_BLANK, CATEGORY_NOT_BLANK,
    CATEGORY_GRAPH, CATEGORY_NOT_GRAPH,
    CATEGORY_PRINT, CATEGORY_NOT_PRINT,
)

MAXUNICODE = 0x10FFFF

# Properties implemented directly by the engine as (positive, negative)
# CATEGORY codes.  The keys are normalised (see _normalize).  digit, space and
# word reuse the \d, \s and \w categories and so are affected by the ASCII,
# LOCALE and UNICODE flags; the rest are plain Unicode properties and are not.
_CATEGORY_PROPERTIES = {
    "digit":      (CATEGORY_DIGIT, CATEGORY_NOT_DIGIT),  # same as \d
    "space":      (CATEGORY_SPACE, CATEGORY_NOT_SPACE),  # same as \s
    # \p{White_Space} is approximated by \s (str.isspace), which also matches
    # the information separators U+001C..U+001F.
    "whitespace": (CATEGORY_SPACE, CATEGORY_NOT_SPACE),
    "wspace":     (CATEGORY_SPACE, CATEGORY_NOT_SPACE),
    "word":       (CATEGORY_WORD,  CATEGORY_NOT_WORD),   # same as \w

    "alphabetic": (CATEGORY_ALPHA, CATEGORY_NOT_ALPHA),
    "alpha":      (CATEGORY_ALPHA, CATEGORY_NOT_ALPHA),  # POSIX
    "lowercase":  (CATEGORY_LOWER, CATEGORY_NOT_LOWER),
    "lower":      (CATEGORY_LOWER, CATEGORY_NOT_LOWER),  # POSIX
    "uppercase":  (CATEGORY_UPPER, CATEGORY_NOT_UPPER),
    "upper":      (CATEGORY_UPPER, CATEGORY_NOT_UPPER),  # POSIX
    "numeric":    (CATEGORY_NUMERIC, CATEGORY_NOT_NUMERIC),
    "printable":  (CATEGORY_PRINTABLE, CATEGORY_NOT_PRINTABLE),
    "cased":      (CATEGORY_CASED, CATEGORY_NOT_CASED),
    "caseignorable": (CATEGORY_CASE_IGNORABLE, CATEGORY_NOT_CASE_IGNORABLE),
    # POSIX classes, the compatibility properties of UTS #18 Annex C (see the
    # compound predicates in sre.c).
    "blank":      (CATEGORY_BLANK, CATEGORY_NOT_BLANK),
    "graph":      (CATEGORY_GRAPH, CATEGORY_NOT_GRAPH),
    "print":      (CATEGORY_PRINT, CATEGORY_NOT_PRINT),
    "assigned":   (CATEGORY_ASSIGNED, CATEGORY_NOT_ASSIGNED),
    "alnum":      (CATEGORY_ALNUM, CATEGORY_NOT_ALNUM),  # POSIX
    "xidstart":    (CATEGORY_XID_START, CATEGORY_NOT_XID_START),
    "xids":        (CATEGORY_XID_START, CATEGORY_NOT_XID_START),
    "xidcontinue": (CATEGORY_XID_CONTINUE, CATEGORY_NOT_XID_CONTINUE),
    "xidc":        (CATEGORY_XID_CONTINUE, CATEGORY_NOT_XID_CONTINUE),
}

# General_Category values matched by an engine category.  CATEGORY_ALPHA
# matches exactly the L group, and CATEGORY_TITLE the Lt category;
# CATEGORY_DIGIT matches Nd (but, like \d, is restricted to ASCII under the
# ASCII flag).  The gc group memberships (L = Lu|Ll|Lt|Lm|Lo, N = Nd|Nl|No)
# are given by the Unicode Standard 4.5, Table 4-4 "General_Category Values"
# (https://www.unicode.org/versions/Unicode18.0.0/core-spec/chapter-4/#G124142)
# and listed in
# https://www.unicode.org/Public/UCD/latest/ucd/PropertyValueAliases.txt
# The compound categories Lu, N, Lm, Nl, No, Cf, Z, Zs, C and Cn are
# combinations of the simple predicates (see sre.c) that reproduce the
# canonical gc partition; they are not Unicode-published identities.
_GC_CATEGORY = {
    "l":  (CATEGORY_ALPHA, CATEGORY_NOT_ALPHA),
    "lt": (CATEGORY_TITLE, CATEGORY_NOT_TITLE),
    "nd": (CATEGORY_DIGIT, CATEGORY_NOT_DIGIT),
    "lu": (CATEGORY_LU, CATEGORY_NOT_LU),
    "n":  (CATEGORY_N, CATEGORY_NOT_N),
    "lm": (CATEGORY_LM, CATEGORY_NOT_LM),
    "nl": (CATEGORY_NL, CATEGORY_NOT_NL),
    "no": (CATEGORY_NO, CATEGORY_NOT_NO),
    "cf": (CATEGORY_CF, CATEGORY_NOT_CF),
    "z":  (CATEGORY_Z, CATEGORY_NOT_Z),
    "zs": (CATEGORY_ZS, CATEGORY_NOT_ZS),
    "c":  (CATEGORY_C, CATEGORY_NOT_C),
    "cn": (CATEGORY_CN, CATEGORY_NOT_CN),
}

# General_Category values whose members are fixed in every Unicode version,
# so they need no table: Cc (control, = POSIX cntrl), Cs (surrogates), Co
# (private use) and the single code points Zl and Zp.  Cc, Cs and Co are the
# control codes, surrogate and private-use areas, fixed by the Unicode
# Standard 23.1, 23.6 and 23.5:
# https://www.unicode.org/versions/Unicode18.0.0/core-spec/chapter-23/
# All five are listed in
# https://www.unicode.org/Public/UCD/latest/ucd/extracted/DerivedGeneralCategory.txt
_CC_RANGES = [(0x00, 0x1F), (0x7F, 0x9F)]
_CS_RANGES = [(0xD800, 0xDFFF)]
_CO_RANGES = [(0xE000, 0xF8FF), (0xF0000, 0xFFFFD), (0x100000, 0x10FFFD)]
_GC_ANALYTIC = {
    "cc": _CC_RANGES,
    "cs": _CS_RANGES,
    "co": _CO_RANGES,
    "zl": [(0x2028, 0x2028)],
    "zp": [(0x2029, 0x2029)],
}

# Pattern_Syntax and Pattern_White_Space are guaranteed immutable by the
# Unicode stability policy, so their members can be hardcoded.
# UAX #31 1.1, "Stability": https://www.unicode.org/reports/tr31/
# Members listed in https://www.unicode.org/Public/UCD/latest/ucd/PropList.txt
_PATTERN_WHITE_SPACE_RANGES = [
    (0x0009, 0x000D), (0x0020, 0x0020), (0x0085, 0x0085), (0x200E, 0x200F),
    (0x2028, 0x2029),
]
_PATTERN_SYNTAX_RANGES = [
    (0x0021, 0x002F), (0x003A, 0x0040), (0x005B, 0x005E), (0x0060, 0x0060),
    (0x007B, 0x007E), (0x00A1, 0x00A7), (0x00A9, 0x00A9), (0x00AB, 0x00AC),
    (0x00AE, 0x00AE), (0x00B0, 0x00B1), (0x00B6, 0x00B6), (0x00BB, 0x00BB),
    (0x00BF, 0x00BF), (0x00D7, 0x00D7), (0x00F7, 0x00F7), (0x2010, 0x2027),
    (0x2030, 0x203E), (0x2041, 0x2053), (0x2055, 0x205E), (0x2190, 0x245F),
    (0x2500, 0x2775), (0x2794, 0x2BFF), (0x2E00, 0x2E7F), (0x3001, 0x3003),
    (0x3008, 0x3020), (0x3030, 0x3030), (0xFD3E, 0xFD3F), (0xFE45, 0xFE46),
]

# Normalised property names that introduce a General_Category value.  A bare
# \p{Lu} is shorthand for \p{gc=Lu} (UTS #18 1.2.4, "Property Syntax").
_GC_KEYS = frozenset({"gc", "generalcategory"})

# Normalised value names for the truth value of a binary property; Yes/No and
# True/False are the binary value aliases of PropertyValueAliases.txt.
_TRUE_VALUES = frozenset({"yes", "y", "true", "t"})
_FALSE_VALUES = frozenset({"no", "n", "false", "f"})


def _analytic_ranges():
    # Properties whose members follow directly from the code point.  Keys are
    # normalised.
    # Noncharacter_Code_Point: U+FDD0..FDEF and the last two of every plane,
    # permanently reserved (the Unicode Standard 23.7, "Noncharacters":
    # https://www.unicode.org/versions/Unicode18.0.0/core-spec/chapter-23/).
    noncharacter = [(0xFDD0, 0xFDEF)]
    noncharacter += [(plane | 0xFFFE, plane | 0xFFFF)
                     for plane in range(0, MAXUNICODE + 1, 0x10000)]
    # Regional_Indicator (RI): the 26 enclosed symbols A..Z, a complete fixed
    # block (PropList.txt binary property).
    regional_indicator = [(0x1F1E6, 0x1F1FF)]
    # ASCII_Hex_Digit (= POSIX xdigit) and Hex_Digit, which adds the fullwidth
    # forms.  Both are complete, fixed sets (PropList.txt binary properties).
    ascii_hex = [(0x30, 0x39), (0x41, 0x46), (0x61, 0x66)]
    hex_digit = ascii_hex + [(0xFF10, 0xFF19), (0xFF21, 0xFF26), (0xFF41, 0xFF46)]
    return {
        "ascii":                 [(0, 0x7F)],
        "any":                   [(0, MAXUNICODE)],
        # Join_Control (U+200C ZWNJ, U+200D ZWJ; the Unicode Standard 23.2,
        # "Layout Controls"), a PropList.txt binary property.
        "joincontrol":           [(0x200C, 0x200D)],
        "regionalindicator":     regional_indicator,
        "ri":                    regional_indicator,
        "noncharactercodepoint": noncharacter,
        "xdigit":                ascii_hex,   # POSIX, ASCII only
        "asciihexdigit":         ascii_hex,
        "ahex":                  ascii_hex,
        "hexdigit":              hex_digit,
        "hex":                   hex_digit,
        # POSIX cntrl is the General_Category Cc, a fixed set of code points.
        "cntrl":                 _CC_RANGES,
        "patternwhitespace":     _PATTERN_WHITE_SPACE_RANGES,
        "patws":                 _PATTERN_WHITE_SPACE_RANGES,
        "patternsyntax":         _PATTERN_SYNTAX_RANGES,
        "patsyn":                _PATTERN_SYNTAX_RANGES,
    }


def _normalize(name):
    # Unicode property and value names are matched loosely: case, spaces,
    # hyphens and underscores are not significant, and an initial "is" prefix
    # is ignored (UAX #44 5.9, "Matching Rules", UAX44-LM3;
    # https://www.unicode.org/reports/tr44/).
    name = name.lower().replace("_", "").replace("-", "").replace(" ", "")
    # Strip a leading "is", unless "is" is the whole name and so not a prefix
    # (e.g. the Line_Break value lb=IS).
    if name != "is":
        name = name.removeprefix("is")
    return name


def _from_ranges(ranges, negate):
    if ranges is None:
        return None
    items = [(LITERAL, lo) if lo == hi else (RANGE, (lo, hi))
             for lo, hi in ranges]
    if negate:
        items.insert(0, (NEGATE, None))
    return (IN, items)


def _general_category(value, negate):
    # Resolve a General_Category value to a subpattern using an engine category
    # or a fixed range set; unsupported values return None.
    cat = _GC_CATEGORY.get(value)
    if cat is not None:
        return (IN, [(CATEGORY, cat[1] if negate else cat[0])])
    return _from_ranges(_GC_ANALYTIC.get(value), negate)


def _truth(value):
    value = _normalize(value)
    if value in _TRUE_VALUES:
        return True
    if value in _FALSE_VALUES:
        return False
    return None


def parse_property(name, negate):
    """Parse the text inside \\p{...} / \\P{...}.

    Return an (IN, items) subpattern, or None if the property is unknown.
    """
    prop, sep, value = name.partition("=")
    if sep:
        key = _normalize(prop)
        if key in _GC_KEYS:
            return _general_category(_normalize(value), negate)
        # A binary property spelled name=yes or name=no.
        truth = _truth(value)
        if truth is None:
            return None
        negate ^= not truth
        cat = _CATEGORY_PROPERTIES.get(key)
        if cat is not None:
            return (IN, [(CATEGORY, cat[1] if negate else cat[0])])
        return _from_ranges(_analytic_ranges().get(key), negate)

    key = _normalize(name)
    cat = _CATEGORY_PROPERTIES.get(key)
    if cat is not None:
        return (IN, [(CATEGORY, cat[1] if negate else cat[0])])
    ranges = _analytic_ranges().get(key)
    if ranges is not None:
        return _from_ranges(ranges, negate)
    return _general_category(key, negate)