| # When adding new tokens, remember to update the PEG generator in |
| # Tools/peg_generator/pegen/parser_generator.py |
| # This will ensure that older versions of Python can generate a Python parser |
| # using "python -m pegen python <GRAMMAR FILE>". |
| |
| ENDMARKER |
| NAME |
| NUMBER |
| STRING |
| NEWLINE |
| INDENT |
| DEDENT |
| |
| LPAR '(' |
| RPAR ')' |
| LSQB '[' |
| RSQB ']' |
| COLON ':' |
| COMMA ',' |
| SEMI ';' |
| PLUS '+' |
| MINUS '-' |
| STAR '*' |
| SLASH '/' |
| VBAR '|' |
| AMPER '&' |
| LESS '<' |
| GREATER '>' |
| EQUAL '=' |
| DOT '.' |
| PERCENT '%' |
| LBRACE '{' |
| RBRACE '}' |
| EQEQUAL '==' |
| NOTEQUAL '!=' |
| LESSEQUAL '<=' |
| GREATEREQUAL '>=' |
| TILDE '~' |
| CIRCUMFLEX '^' |
| LEFTSHIFT '<<' |
| RIGHTSHIFT '>>' |
| DOUBLESTAR '**' |
| PLUSEQUAL '+=' |
| MINEQUAL '-=' |
| STAREQUAL '*=' |
| SLASHEQUAL '/=' |
| PERCENTEQUAL '%=' |
| AMPEREQUAL '&=' |
| VBAREQUAL '|=' |
| CIRCUMFLEXEQUAL '^=' |
| LEFTSHIFTEQUAL '<<=' |
| RIGHTSHIFTEQUAL '>>=' |
| DOUBLESTAREQUAL '**=' |
| DOUBLESLASH '//' |
| DOUBLESLASHEQUAL '//=' |
| AT '@' |
| ATEQUAL '@=' |
| RARROW '->' |
| ELLIPSIS '...' |
| COLONEQUAL ':=' |
| EXCLAMATION '!' |
| |
| OP |
| TYPE_IGNORE |
| TYPE_COMMENT |
| SOFT_KEYWORD |
| FSTRING_START |
| FSTRING_MIDDLE |
| FSTRING_END |
| TSTRING_START |
| TSTRING_MIDDLE |
| TSTRING_END |
| COMMENT |
| NL |
| ERRORTOKEN |
| |
| # These aren't used by the C tokenizer but are needed for tokenize.py |
| ENCODING |
