blob: 63563ceb7330ea399deb7cbc80d9ac6eb8ff8481 [file] [log] [blame]
/*
* This file includes functions to transform a concrete syntax tree (CST) to
* an abstract syntax tree (AST). The main function is PyAST_FromNode().
*
*/
#include "Python.h"
#include "Python-ast.h"
#include "node.h"
#include "ast.h"
#include "token.h"
#include "pythonrun.h"
#include <assert.h>
#include <stdbool.h>
#define MAXLEVEL 200 /* Max parentheses level */
static int validate_stmts(asdl_seq *);
static int validate_exprs(asdl_seq *, expr_context_ty, int);
static int validate_nonempty_seq(asdl_seq *, const char *, const char *);
static int validate_stmt(stmt_ty);
static int validate_expr(expr_ty, expr_context_ty);
static int
validate_name(PyObject *name)
{
assert(PyUnicode_Check(name));
static const char * const forbidden[] = {
"None",
"True",
"False",
NULL
};
for (int i = 0; forbidden[i] != NULL; i++) {
if (_PyUnicode_EqualToASCIIString(name, forbidden[i])) {
PyErr_Format(PyExc_ValueError, "Name node can't be used with '%s' constant", forbidden[i]);
return 0;
}
}
return 1;
}
static int
validate_comprehension(asdl_seq *gens)
{
Py_ssize_t i;
if (!asdl_seq_LEN(gens)) {
PyErr_SetString(PyExc_ValueError, "comprehension with no generators");
return 0;
}
for (i = 0; i < asdl_seq_LEN(gens); i++) {
comprehension_ty comp = asdl_seq_GET(gens, i);
if (!validate_expr(comp->target, Store) ||
!validate_expr(comp->iter, Load) ||
!validate_exprs(comp->ifs, Load, 0))
return 0;
}
return 1;
}
static int
validate_slice(slice_ty slice)
{
switch (slice->kind) {
case Slice_kind:
return (!slice->v.Slice.lower || validate_expr(slice->v.Slice.lower, Load)) &&
(!slice->v.Slice.upper || validate_expr(slice->v.Slice.upper, Load)) &&
(!slice->v.Slice.step || validate_expr(slice->v.Slice.step, Load));
case ExtSlice_kind: {
Py_ssize_t i;
if (!validate_nonempty_seq(slice->v.ExtSlice.dims, "dims", "ExtSlice"))
return 0;
for (i = 0; i < asdl_seq_LEN(slice->v.ExtSlice.dims); i++)
if (!validate_slice(asdl_seq_GET(slice->v.ExtSlice.dims, i)))
return 0;
return 1;
}
case Index_kind:
return validate_expr(slice->v.Index.value, Load);
default:
PyErr_SetString(PyExc_SystemError, "unknown slice node");
return 0;
}
}
static int
validate_keywords(asdl_seq *keywords)
{
Py_ssize_t i;
for (i = 0; i < asdl_seq_LEN(keywords); i++)
if (!validate_expr(((keyword_ty)asdl_seq_GET(keywords, i))->value, Load))
return 0;
return 1;
}
static int
validate_args(asdl_seq *args)
{
Py_ssize_t i;
for (i = 0; i < asdl_seq_LEN(args); i++) {
arg_ty arg = asdl_seq_GET(args, i);
if (arg->annotation && !validate_expr(arg->annotation, Load))
return 0;
}
return 1;
}
static const char *
expr_context_name(expr_context_ty ctx)
{
switch (ctx) {
case Load:
return "Load";
case Store:
return "Store";
case Del:
return "Del";
case AugLoad:
return "AugLoad";
case AugStore:
return "AugStore";
case Param:
return "Param";
default:
Py_UNREACHABLE();
}
}
static int
validate_arguments(arguments_ty args)
{
if (!validate_args(args->posonlyargs) || !validate_args(args->args)) {
return 0;
}
if (args->vararg && args->vararg->annotation
&& !validate_expr(args->vararg->annotation, Load)) {
return 0;
}
if (!validate_args(args->kwonlyargs))
return 0;
if (args->kwarg && args->kwarg->annotation
&& !validate_expr(args->kwarg->annotation, Load)) {
return 0;
}
if (asdl_seq_LEN(args->defaults) > asdl_seq_LEN(args->posonlyargs) + asdl_seq_LEN(args->args)) {
PyErr_SetString(PyExc_ValueError, "more positional defaults than args on arguments");
return 0;
}
if (asdl_seq_LEN(args->kw_defaults) != asdl_seq_LEN(args->kwonlyargs)) {
PyErr_SetString(PyExc_ValueError, "length of kwonlyargs is not the same as "
"kw_defaults on arguments");
return 0;
}
return validate_exprs(args->defaults, Load, 0) && validate_exprs(args->kw_defaults, Load, 1);
}
static int
validate_constant(PyObject *value)
{
if (value == Py_None || value == Py_Ellipsis)
return 1;
if (PyLong_CheckExact(value)
|| PyFloat_CheckExact(value)
|| PyComplex_CheckExact(value)
|| PyBool_Check(value)
|| PyUnicode_CheckExact(value)
|| PyBytes_CheckExact(value))
return 1;
if (PyTuple_CheckExact(value) || PyFrozenSet_CheckExact(value)) {
PyObject *it;
it = PyObject_GetIter(value);
if (it == NULL)
return 0;
while (1) {
PyObject *item = PyIter_Next(it);
if (item == NULL) {
if (PyErr_Occurred()) {
Py_DECREF(it);
return 0;
}
break;
}
if (!validate_constant(item)) {
Py_DECREF(it);
Py_DECREF(item);
return 0;
}
Py_DECREF(item);
}
Py_DECREF(it);
return 1;
}
return 0;
}
static int
validate_expr(expr_ty exp, expr_context_ty ctx)
{
int check_ctx = 1;
expr_context_ty actual_ctx;
/* First check expression context. */
switch (exp->kind) {
case Attribute_kind:
actual_ctx = exp->v.Attribute.ctx;
break;
case Subscript_kind:
actual_ctx = exp->v.Subscript.ctx;
break;
case Starred_kind:
actual_ctx = exp->v.Starred.ctx;
break;
case Name_kind:
if (!validate_name(exp->v.Name.id)) {
return 0;
}
actual_ctx = exp->v.Name.ctx;
break;
case List_kind:
actual_ctx = exp->v.List.ctx;
break;
case Tuple_kind:
actual_ctx = exp->v.Tuple.ctx;
break;
default:
if (ctx != Load) {
PyErr_Format(PyExc_ValueError, "expression which can't be "
"assigned to in %s context", expr_context_name(ctx));
return 0;
}
check_ctx = 0;
/* set actual_ctx to prevent gcc warning */
actual_ctx = 0;
}
if (check_ctx && actual_ctx != ctx) {
PyErr_Format(PyExc_ValueError, "expression must have %s context but has %s instead",
expr_context_name(ctx), expr_context_name(actual_ctx));
return 0;
}
/* Now validate expression. */
switch (exp->kind) {
case BoolOp_kind:
if (asdl_seq_LEN(exp->v.BoolOp.values) < 2) {
PyErr_SetString(PyExc_ValueError, "BoolOp with less than 2 values");
return 0;
}
return validate_exprs(exp->v.BoolOp.values, Load, 0);
case BinOp_kind:
return validate_expr(exp->v.BinOp.left, Load) &&
validate_expr(exp->v.BinOp.right, Load);
case UnaryOp_kind:
return validate_expr(exp->v.UnaryOp.operand, Load);
case Lambda_kind:
return validate_arguments(exp->v.Lambda.args) &&
validate_expr(exp->v.Lambda.body, Load);
case IfExp_kind:
return validate_expr(exp->v.IfExp.test, Load) &&
validate_expr(exp->v.IfExp.body, Load) &&
validate_expr(exp->v.IfExp.orelse, Load);
case Dict_kind:
if (asdl_seq_LEN(exp->v.Dict.keys) != asdl_seq_LEN(exp->v.Dict.values)) {
PyErr_SetString(PyExc_ValueError,
"Dict doesn't have the same number of keys as values");
return 0;
}
/* null_ok=1 for keys expressions to allow dict unpacking to work in
dict literals, i.e. ``{**{a:b}}`` */
return validate_exprs(exp->v.Dict.keys, Load, /*null_ok=*/ 1) &&
validate_exprs(exp->v.Dict.values, Load, /*null_ok=*/ 0);
case Set_kind:
return validate_exprs(exp->v.Set.elts, Load, 0);
#define COMP(NAME) \
case NAME ## _kind: \
return validate_comprehension(exp->v.NAME.generators) && \
validate_expr(exp->v.NAME.elt, Load);
COMP(ListComp)
COMP(SetComp)
COMP(GeneratorExp)
#undef COMP
case DictComp_kind:
return validate_comprehension(exp->v.DictComp.generators) &&
validate_expr(exp->v.DictComp.key, Load) &&
validate_expr(exp->v.DictComp.value, Load);
case Yield_kind:
return !exp->v.Yield.value || validate_expr(exp->v.Yield.value, Load);
case YieldFrom_kind:
return validate_expr(exp->v.YieldFrom.value, Load);
case Await_kind:
return validate_expr(exp->v.Await.value, Load);
case Compare_kind:
if (!asdl_seq_LEN(exp->v.Compare.comparators)) {
PyErr_SetString(PyExc_ValueError, "Compare with no comparators");
return 0;
}
if (asdl_seq_LEN(exp->v.Compare.comparators) !=
asdl_seq_LEN(exp->v.Compare.ops)) {
PyErr_SetString(PyExc_ValueError, "Compare has a different number "
"of comparators and operands");
return 0;
}
return validate_exprs(exp->v.Compare.comparators, Load, 0) &&
validate_expr(exp->v.Compare.left, Load);
case Call_kind:
return validate_expr(exp->v.Call.func, Load) &&
validate_exprs(exp->v.Call.args, Load, 0) &&
validate_keywords(exp->v.Call.keywords);
case Constant_kind:
if (!validate_constant(exp->v.Constant.value)) {
PyErr_Format(PyExc_TypeError,
"got an invalid type in Constant: %s",
Py_TYPE(exp->v.Constant.value)->tp_name);
return 0;
}
return 1;
case JoinedStr_kind:
return validate_exprs(exp->v.JoinedStr.values, Load, 0);
case FormattedValue_kind:
if (validate_expr(exp->v.FormattedValue.value, Load) == 0)
return 0;
if (exp->v.FormattedValue.format_spec)
return validate_expr(exp->v.FormattedValue.format_spec, Load);
return 1;
case Attribute_kind:
return validate_expr(exp->v.Attribute.value, Load);
case Subscript_kind:
return validate_slice(exp->v.Subscript.slice) &&
validate_expr(exp->v.Subscript.value, Load);
case Starred_kind:
return validate_expr(exp->v.Starred.value, ctx);
case List_kind:
return validate_exprs(exp->v.List.elts, ctx, 0);
case Tuple_kind:
return validate_exprs(exp->v.Tuple.elts, ctx, 0);
case NamedExpr_kind:
return validate_expr(exp->v.NamedExpr.value, Load);
/* This last case doesn't have any checking. */
case Name_kind:
return 1;
}
PyErr_SetString(PyExc_SystemError, "unexpected expression");
return 0;
}
static int
validate_nonempty_seq(asdl_seq *seq, const char *what, const char *owner)
{
if (asdl_seq_LEN(seq))
return 1;
PyErr_Format(PyExc_ValueError, "empty %s on %s", what, owner);
return 0;
}
static int
validate_assignlist(asdl_seq *targets, expr_context_ty ctx)
{
return validate_nonempty_seq(targets, "targets", ctx == Del ? "Delete" : "Assign") &&
validate_exprs(targets, ctx, 0);
}
static int
validate_body(asdl_seq *body, const char *owner)
{
return validate_nonempty_seq(body, "body", owner) && validate_stmts(body);
}
static int
validate_stmt(stmt_ty stmt)
{
Py_ssize_t i;
switch (stmt->kind) {
case FunctionDef_kind:
return validate_body(stmt->v.FunctionDef.body, "FunctionDef") &&
validate_arguments(stmt->v.FunctionDef.args) &&
validate_exprs(stmt->v.FunctionDef.decorator_list, Load, 0) &&
(!stmt->v.FunctionDef.returns ||
validate_expr(stmt->v.FunctionDef.returns, Load));
case ClassDef_kind:
return validate_body(stmt->v.ClassDef.body, "ClassDef") &&
validate_exprs(stmt->v.ClassDef.bases, Load, 0) &&
validate_keywords(stmt->v.ClassDef.keywords) &&
validate_exprs(stmt->v.ClassDef.decorator_list, Load, 0);
case Return_kind:
return !stmt->v.Return.value || validate_expr(stmt->v.Return.value, Load);
case Delete_kind:
return validate_assignlist(stmt->v.Delete.targets, Del);
case Assign_kind:
return validate_assignlist(stmt->v.Assign.targets, Store) &&
validate_expr(stmt->v.Assign.value, Load);
case AugAssign_kind:
return validate_expr(stmt->v.AugAssign.target, Store) &&
validate_expr(stmt->v.AugAssign.value, Load);
case AnnAssign_kind:
if (stmt->v.AnnAssign.target->kind != Name_kind &&
stmt->v.AnnAssign.simple) {
PyErr_SetString(PyExc_TypeError,
"AnnAssign with simple non-Name target");
return 0;
}
return validate_expr(stmt->v.AnnAssign.target, Store) &&
(!stmt->v.AnnAssign.value ||
validate_expr(stmt->v.AnnAssign.value, Load)) &&
validate_expr(stmt->v.AnnAssign.annotation, Load);
case For_kind:
return validate_expr(stmt->v.For.target, Store) &&
validate_expr(stmt->v.For.iter, Load) &&
validate_body(stmt->v.For.body, "For") &&
validate_stmts(stmt->v.For.orelse);
case AsyncFor_kind:
return validate_expr(stmt->v.AsyncFor.target, Store) &&
validate_expr(stmt->v.AsyncFor.iter, Load) &&
validate_body(stmt->v.AsyncFor.body, "AsyncFor") &&
validate_stmts(stmt->v.AsyncFor.orelse);
case While_kind:
return validate_expr(stmt->v.While.test, Load) &&
validate_body(stmt->v.While.body, "While") &&
validate_stmts(stmt->v.While.orelse);
case If_kind:
return validate_expr(stmt->v.If.test, Load) &&
validate_body(stmt->v.If.body, "If") &&
validate_stmts(stmt->v.If.orelse);
case With_kind:
if (!validate_nonempty_seq(stmt->v.With.items, "items", "With"))
return 0;
for (i = 0; i < asdl_seq_LEN(stmt->v.With.items); i++) {
withitem_ty item = asdl_seq_GET(stmt->v.With.items, i);
if (!validate_expr(item->context_expr, Load) ||
(item->optional_vars && !validate_expr(item->optional_vars, Store)))
return 0;
}
return validate_body(stmt->v.With.body, "With");
case AsyncWith_kind:
if (!validate_nonempty_seq(stmt->v.AsyncWith.items, "items", "AsyncWith"))
return 0;
for (i = 0; i < asdl_seq_LEN(stmt->v.AsyncWith.items); i++) {
withitem_ty item = asdl_seq_GET(stmt->v.AsyncWith.items, i);
if (!validate_expr(item->context_expr, Load) ||
(item->optional_vars && !validate_expr(item->optional_vars, Store)))
return 0;
}
return validate_body(stmt->v.AsyncWith.body, "AsyncWith");
case Raise_kind:
if (stmt->v.Raise.exc) {
return validate_expr(stmt->v.Raise.exc, Load) &&
(!stmt->v.Raise.cause || validate_expr(stmt->v.Raise.cause, Load));
}
if (stmt->v.Raise.cause) {
PyErr_SetString(PyExc_ValueError, "Raise with cause but no exception");
return 0;
}
return 1;
case Try_kind:
if (!validate_body(stmt->v.Try.body, "Try"))
return 0;
if (!asdl_seq_LEN(stmt->v.Try.handlers) &&
!asdl_seq_LEN(stmt->v.Try.finalbody)) {
PyErr_SetString(PyExc_ValueError, "Try has neither except handlers nor finalbody");
return 0;
}
if (!asdl_seq_LEN(stmt->v.Try.handlers) &&
asdl_seq_LEN(stmt->v.Try.orelse)) {
PyErr_SetString(PyExc_ValueError, "Try has orelse but no except handlers");
return 0;
}
for (i = 0; i < asdl_seq_LEN(stmt->v.Try.handlers); i++) {
excepthandler_ty handler = asdl_seq_GET(stmt->v.Try.handlers, i);
if ((handler->v.ExceptHandler.type &&
!validate_expr(handler->v.ExceptHandler.type, Load)) ||
!validate_body(handler->v.ExceptHandler.body, "ExceptHandler"))
return 0;
}
return (!asdl_seq_LEN(stmt->v.Try.finalbody) ||
validate_stmts(stmt->v.Try.finalbody)) &&
(!asdl_seq_LEN(stmt->v.Try.orelse) ||
validate_stmts(stmt->v.Try.orelse));
case Assert_kind:
return validate_expr(stmt->v.Assert.test, Load) &&
(!stmt->v.Assert.msg || validate_expr(stmt->v.Assert.msg, Load));
case Import_kind:
return validate_nonempty_seq(stmt->v.Import.names, "names", "Import");
case ImportFrom_kind:
if (stmt->v.ImportFrom.level < 0) {
PyErr_SetString(PyExc_ValueError, "Negative ImportFrom level");
return 0;
}
return validate_nonempty_seq(stmt->v.ImportFrom.names, "names", "ImportFrom");
case Global_kind:
return validate_nonempty_seq(stmt->v.Global.names, "names", "Global");
case Nonlocal_kind:
return validate_nonempty_seq(stmt->v.Nonlocal.names, "names", "Nonlocal");
case Expr_kind:
return validate_expr(stmt->v.Expr.value, Load);
case AsyncFunctionDef_kind:
return validate_body(stmt->v.AsyncFunctionDef.body, "AsyncFunctionDef") &&
validate_arguments(stmt->v.AsyncFunctionDef.args) &&
validate_exprs(stmt->v.AsyncFunctionDef.decorator_list, Load, 0) &&
(!stmt->v.AsyncFunctionDef.returns ||
validate_expr(stmt->v.AsyncFunctionDef.returns, Load));
case Pass_kind:
case Break_kind:
case Continue_kind:
return 1;
default:
PyErr_SetString(PyExc_SystemError, "unexpected statement");
return 0;
}
}
static int
validate_stmts(asdl_seq *seq)
{
Py_ssize_t i;
for (i = 0; i < asdl_seq_LEN(seq); i++) {
stmt_ty stmt = asdl_seq_GET(seq, i);
if (stmt) {
if (!validate_stmt(stmt))
return 0;
}
else {
PyErr_SetString(PyExc_ValueError,
"None disallowed in statement list");
return 0;
}
}
return 1;
}
static int
validate_exprs(asdl_seq *exprs, expr_context_ty ctx, int null_ok)
{
Py_ssize_t i;
for (i = 0; i < asdl_seq_LEN(exprs); i++) {
expr_ty expr = asdl_seq_GET(exprs, i);
if (expr) {
if (!validate_expr(expr, ctx))
return 0;
}
else if (!null_ok) {
PyErr_SetString(PyExc_ValueError,
"None disallowed in expression list");
return 0;
}
}
return 1;
}
int
PyAST_Validate(mod_ty mod)
{
int res = 0;
switch (mod->kind) {
case Module_kind:
res = validate_stmts(mod->v.Module.body);
break;
case Interactive_kind:
res = validate_stmts(mod->v.Interactive.body);
break;
case Expression_kind:
res = validate_expr(mod->v.Expression.body, Load);
break;
case Suite_kind:
PyErr_SetString(PyExc_ValueError, "Suite is not valid in the CPython compiler");
break;
default:
PyErr_SetString(PyExc_SystemError, "impossible module node");
res = 0;
break;
}
return res;
}
/* This is done here, so defines like "test" don't interfere with AST use above. */
#include "grammar.h"
#include "parsetok.h"
#include "graminit.h"
/* Data structure used internally */
struct compiling {
PyArena *c_arena; /* Arena for allocating memory. */
PyObject *c_filename; /* filename */
PyObject *c_normalize; /* Normalization function from unicodedata. */
int c_feature_version; /* Latest minor version of Python for allowed features */
};
static asdl_seq *seq_for_testlist(struct compiling *, const node *);
static expr_ty ast_for_expr(struct compiling *, const node *);
static stmt_ty ast_for_stmt(struct compiling *, const node *);
static asdl_seq *ast_for_suite(struct compiling *c, const node *n);
static asdl_seq *ast_for_exprlist(struct compiling *, const node *,
expr_context_ty);
static expr_ty ast_for_testlist(struct compiling *, const node *);
static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *);
static stmt_ty ast_for_with_stmt(struct compiling *, const node *, bool);
static stmt_ty ast_for_for_stmt(struct compiling *, const node *, bool);
/* Note different signature for ast_for_call */
static expr_ty ast_for_call(struct compiling *, const node *, expr_ty,
const node *, const node *, const node *);
static PyObject *parsenumber(struct compiling *, const char *);
static expr_ty parsestrplus(struct compiling *, const node *n);
static void get_last_end_pos(asdl_seq *, int *, int *);
#define COMP_GENEXP 0
#define COMP_LISTCOMP 1
#define COMP_SETCOMP 2
static int
init_normalization(struct compiling *c)
{
PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
if (!m)
return 0;
c->c_normalize = PyObject_GetAttrString(m, "normalize");
Py_DECREF(m);
if (!c->c_normalize)
return 0;
return 1;
}
static identifier
new_identifier(const char *n, struct compiling *c)
{
PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
if (!id)
return NULL;
/* PyUnicode_DecodeUTF8 should always return a ready string. */
assert(PyUnicode_IS_READY(id));
/* Check whether there are non-ASCII characters in the
identifier; if so, normalize to NFKC. */
if (!PyUnicode_IS_ASCII(id)) {
PyObject *id2;
_Py_IDENTIFIER(NFKC);
if (!c->c_normalize && !init_normalization(c)) {
Py_DECREF(id);
return NULL;
}
PyObject *form = _PyUnicode_FromId(&PyId_NFKC);
if (form == NULL) {
Py_DECREF(id);
return NULL;
}
PyObject *args[2] = {form, id};
id2 = _PyObject_FastCall(c->c_normalize, args, 2);
Py_DECREF(id);
if (!id2)
return NULL;
if (!PyUnicode_Check(id2)) {
PyErr_Format(PyExc_TypeError,
"unicodedata.normalize() must return a string, not "
"%.200s",
Py_TYPE(id2)->tp_name);
Py_DECREF(id2);
return NULL;
}
id = id2;
}
PyUnicode_InternInPlace(&id);
if (PyArena_AddPyObject(c->c_arena, id) < 0) {
Py_DECREF(id);
return NULL;
}
return id;
}
#define NEW_IDENTIFIER(n) new_identifier(STR(n), c)
static int
ast_error(struct compiling *c, const node *n, const char *errmsg, ...)
{
PyObject *value, *errstr, *loc, *tmp;
va_list va;
va_start(va, errmsg);
errstr = PyUnicode_FromFormatV(errmsg, va);
va_end(va);
if (!errstr) {
return 0;
}
loc = PyErr_ProgramTextObject(c->c_filename, LINENO(n));
if (!loc) {
Py_INCREF(Py_None);
loc = Py_None;
}
tmp = Py_BuildValue("(OiiN)", c->c_filename, LINENO(n), n->n_col_offset + 1, loc);
if (!tmp) {
Py_DECREF(errstr);
return 0;
}
value = PyTuple_Pack(2, errstr, tmp);
Py_DECREF(errstr);
Py_DECREF(tmp);
if (value) {
PyErr_SetObject(PyExc_SyntaxError, value);
Py_DECREF(value);
}
return 0;
}
/* num_stmts() returns number of contained statements.
Use this routine to determine how big a sequence is needed for
the statements in a parse tree. Its raison d'etre is this bit of
grammar:
stmt: simple_stmt | compound_stmt
simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
A simple_stmt can contain multiple small_stmt elements joined
by semicolons. If the arg is a simple_stmt, the number of
small_stmt elements is returned.
*/
static string
new_type_comment(const char *s, struct compiling *c)
{
PyObject *res = PyUnicode_DecodeUTF8(s, strlen(s), NULL);
if (res == NULL)
return NULL;
if (PyArena_AddPyObject(c->c_arena, res) < 0) {
Py_DECREF(res);
return NULL;
}
return res;
}
#define NEW_TYPE_COMMENT(n) new_type_comment(STR(n), c)
static int
num_stmts(const node *n)
{
int i, l;
node *ch;
switch (TYPE(n)) {
case single_input:
if (TYPE(CHILD(n, 0)) == NEWLINE)
return 0;
else
return num_stmts(CHILD(n, 0));
case file_input:
l = 0;
for (i = 0; i < NCH(n); i++) {
ch = CHILD(n, i);
if (TYPE(ch) == stmt)
l += num_stmts(ch);
}
return l;
case stmt:
return num_stmts(CHILD(n, 0));
case compound_stmt:
return 1;
case simple_stmt:
return NCH(n) / 2; /* Divide by 2 to remove count of semi-colons */
case suite:
case func_body_suite:
/* func_body_suite: simple_stmt | NEWLINE [TYPE_COMMENT NEWLINE] INDENT stmt+ DEDENT */
/* suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT */
if (NCH(n) == 1)
return num_stmts(CHILD(n, 0));
else {
i = 2;
l = 0;
if (TYPE(CHILD(n, 1)) == TYPE_COMMENT)
i += 2;
for (; i < (NCH(n) - 1); i++)
l += num_stmts(CHILD(n, i));
return l;
}
default: {
char buf[128];
sprintf(buf, "Non-statement found: %d %d",
TYPE(n), NCH(n));
Py_FatalError(buf);
}
}
Py_UNREACHABLE();
}
/* Transform the CST rooted at node * to the appropriate AST
*/
mod_ty
PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags,
PyObject *filename, PyArena *arena)
{
int i, j, k, num;
asdl_seq *stmts = NULL;
asdl_seq *type_ignores = NULL;
stmt_ty s;
node *ch;
struct compiling c;
mod_ty res = NULL;
asdl_seq *argtypes = NULL;
expr_ty ret, arg;
c.c_arena = arena;
/* borrowed reference */
c.c_filename = filename;
c.c_normalize = NULL;
c.c_feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ?
flags->cf_feature_version : PY_MINOR_VERSION;
if (TYPE(n) == encoding_decl)
n = CHILD(n, 0);
k = 0;
switch (TYPE(n)) {
case file_input:
stmts = _Py_asdl_seq_new(num_stmts(n), arena);
if (!stmts)
goto out;
for (i = 0; i < NCH(n) - 1; i++) {
ch = CHILD(n, i);
if (TYPE(ch) == NEWLINE)
continue;
REQ(ch, stmt);
num = num_stmts(ch);
if (num == 1) {
s = ast_for_stmt(&c, ch);
if (!s)
goto out;
asdl_seq_SET(stmts, k++, s);
}
else {
ch = CHILD(ch, 0);
REQ(ch, simple_stmt);
for (j = 0; j < num; j++) {
s = ast_for_stmt(&c, CHILD(ch, j * 2));
if (!s)
goto out;
asdl_seq_SET(stmts, k++, s);
}
}
}
/* Type ignores are stored under the ENDMARKER in file_input. */
ch = CHILD(n, NCH(n) - 1);
REQ(ch, ENDMARKER);
num = NCH(ch);
type_ignores = _Py_asdl_seq_new(num, arena);
if (!type_ignores)
goto out;
for (i = 0; i < num; i++) {
string type_comment = new_type_comment(STR(CHILD(ch, i)), &c);
if (!type_comment)
goto out;
type_ignore_ty ti = TypeIgnore(LINENO(CHILD(ch, i)), type_comment, arena);
if (!ti)
goto out;
asdl_seq_SET(type_ignores, i, ti);
}
res = Module(stmts, type_ignores, arena);
break;
case eval_input: {
expr_ty testlist_ast;
/* XXX Why not comp_for here? */
testlist_ast = ast_for_testlist(&c, CHILD(n, 0));
if (!testlist_ast)
goto out;
res = Expression(testlist_ast, arena);
break;
}
case single_input:
if (TYPE(CHILD(n, 0)) == NEWLINE) {
stmts = _Py_asdl_seq_new(1, arena);
if (!stmts)
goto out;
asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset,
arena));
if (!asdl_seq_GET(stmts, 0))
goto out;
res = Interactive(stmts, arena);
}
else {
n = CHILD(n, 0);
num = num_stmts(n);
stmts = _Py_asdl_seq_new(num, arena);
if (!stmts)
goto out;
if (num == 1) {
s = ast_for_stmt(&c, n);
if (!s)
goto out;
asdl_seq_SET(stmts, 0, s);
}
else {
/* Only a simple_stmt can contain multiple statements. */
REQ(n, simple_stmt);
for (i = 0; i < NCH(n); i += 2) {
if (TYPE(CHILD(n, i)) == NEWLINE)
break;
s = ast_for_stmt(&c, CHILD(n, i));
if (!s)
goto out;
asdl_seq_SET(stmts, i / 2, s);
}
}
res = Interactive(stmts, arena);
}
break;
case func_type_input:
n = CHILD(n, 0);
REQ(n, func_type);
if (TYPE(CHILD(n, 1)) == typelist) {
ch = CHILD(n, 1);
/* this is overly permissive -- we don't pay any attention to
* stars on the args -- just parse them into an ordered list */
num = 0;
for (i = 0; i < NCH(ch); i++) {
if (TYPE(CHILD(ch, i)) == test) {
num++;
}
}
argtypes = _Py_asdl_seq_new(num, arena);
if (!argtypes)
goto out;
j = 0;
for (i = 0; i < NCH(ch); i++) {
if (TYPE(CHILD(ch, i)) == test) {
arg = ast_for_expr(&c, CHILD(ch, i));
if (!arg)
goto out;
asdl_seq_SET(argtypes, j++, arg);
}
}
}
else {
argtypes = _Py_asdl_seq_new(0, arena);
if (!argtypes)
goto out;
}
ret = ast_for_expr(&c, CHILD(n, NCH(n) - 1));
if (!ret)
goto out;
res = FunctionType(argtypes, ret, arena);
break;
default:
PyErr_Format(PyExc_SystemError,
"invalid node %d for PyAST_FromNode", TYPE(n));
goto out;
}
out:
if (c.c_normalize) {
Py_DECREF(c.c_normalize);
}
return res;
}
mod_ty
PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename_str,
PyArena *arena)
{
mod_ty mod;
PyObject *filename;
filename = PyUnicode_DecodeFSDefault(filename_str);
if (filename == NULL)
return NULL;
mod = PyAST_FromNodeObject(n, flags, filename, arena);
Py_DECREF(filename);
return mod;
}
/* Return the AST repr. of the operator represented as syntax (|, ^, etc.)
*/
static operator_ty
get_operator(struct compiling *c, const node *n)
{
switch (TYPE(n)) {
case VBAR:
return BitOr;
case CIRCUMFLEX:
return BitXor;
case AMPER:
return BitAnd;
case LEFTSHIFT:
return LShift;
case RIGHTSHIFT:
return RShift;
case PLUS:
return Add;
case MINUS:
return Sub;
case STAR:
return Mult;
case AT:
if (c->c_feature_version < 5) {
ast_error(c, n,
"The '@' operator is only supported in Python 3.5 and greater");
return (operator_ty)0;
}
return MatMult;
case SLASH:
return Div;
case DOUBLESLASH:
return FloorDiv;
case PERCENT:
return Mod;
default:
return (operator_ty)0;
}
}
static const char * const FORBIDDEN[] = {
"None",
"True",
"False",
"__debug__",
NULL,
};
static int
forbidden_name(struct compiling *c, identifier name, const node *n,
int full_checks)
{
assert(PyUnicode_Check(name));
const char * const *p = FORBIDDEN;
if (!full_checks) {
/* In most cases, the parser will protect True, False, and None
from being assign to. */
p += 3;
}
for (; *p; p++) {
if (_PyUnicode_EqualToASCIIString(name, *p)) {
ast_error(c, n, "cannot assign to %U", name);
return 1;
}
}
return 0;
}
static expr_ty
copy_location(expr_ty e, const node *n, const node *end)
{
if (e) {
e->lineno = LINENO(n);
e->col_offset = n->n_col_offset;
e->end_lineno = end->n_end_lineno;
e->end_col_offset = end->n_end_col_offset;
}
return e;
}
static const char *
get_expr_name(expr_ty e)
{
switch (e->kind) {
case Attribute_kind:
return "attribute";
case Subscript_kind:
return "subscript";
case Starred_kind:
return "starred";
case Name_kind:
return "name";
case List_kind:
return "list";
case Tuple_kind:
return "tuple";
case Lambda_kind:
return "lambda";
case Call_kind:
return "function call";
case BoolOp_kind:
case BinOp_kind:
case UnaryOp_kind:
return "operator";
case GeneratorExp_kind:
return "generator expression";
case Yield_kind:
case YieldFrom_kind:
return "yield expression";
case Await_kind:
return "await expression";
case ListComp_kind:
return "list comprehension";
case SetComp_kind:
return "set comprehension";
case DictComp_kind:
return "dict comprehension";
case Dict_kind:
return "dict display";
case Set_kind:
return "set display";
case JoinedStr_kind:
case FormattedValue_kind:
return "f-string expression";
case Constant_kind: {
PyObject *value = e->v.Constant.value;
if (value == Py_None) {
return "None";
}
if (value == Py_False) {
return "False";
}
if (value == Py_True) {
return "True";
}
if (value == Py_Ellipsis) {
return "Ellipsis";
}
return "literal";
}
case Compare_kind:
return "comparison";
case IfExp_kind:
return "conditional expression";
case NamedExpr_kind:
return "named expression";
default:
PyErr_Format(PyExc_SystemError,
"unexpected expression in assignment %d (line %d)",
e->kind, e->lineno);
return NULL;
}
}
/* Set the context ctx for expr_ty e, recursively traversing e.
Only sets context for expr kinds that "can appear in assignment context"
(according to ../Parser/Python.asdl). For other expr kinds, it sets
an appropriate syntax error and returns false.
*/
static int
set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n)
{
asdl_seq *s = NULL;
/* The ast defines augmented store and load contexts, but the
implementation here doesn't actually use them. The code may be
a little more complex than necessary as a result. It also means
that expressions in an augmented assignment have a Store context.
Consider restructuring so that augmented assignment uses
set_context(), too.
*/
assert(ctx != AugStore && ctx != AugLoad);
switch (e->kind) {
case Attribute_kind:
e->v.Attribute.ctx = ctx;
if (ctx == Store && forbidden_name(c, e->v.Attribute.attr, n, 1))
return 0;
break;
case Subscript_kind:
e->v.Subscript.ctx = ctx;
break;
case Starred_kind:
e->v.Starred.ctx = ctx;
if (!set_context(c, e->v.Starred.value, ctx, n))
return 0;
break;
case Name_kind:
if (ctx == Store) {
if (forbidden_name(c, e->v.Name.id, n, 0))
return 0; /* forbidden_name() calls ast_error() */
}
e->v.Name.ctx = ctx;
break;
case List_kind:
e->v.List.ctx = ctx;
s = e->v.List.elts;
break;
case Tuple_kind:
e->v.Tuple.ctx = ctx;
s = e->v.Tuple.elts;
break;
default: {
const char *expr_name = get_expr_name(e);
if (expr_name != NULL) {
ast_error(c, n, "cannot %s %s",
ctx == Store ? "assign to" : "delete",
expr_name);
}
return 0;
}
}
/* If the LHS is a list or tuple, we need to set the assignment
context for all the contained elements.
*/
if (s) {
Py_ssize_t i;
for (i = 0; i < asdl_seq_LEN(s); i++) {
if (!set_context(c, (expr_ty)asdl_seq_GET(s, i), ctx, n))
return 0;
}
}
return 1;
}
static operator_ty
ast_for_augassign(struct compiling *c, const node *n)
{
REQ(n, augassign);
n = CHILD(n, 0);
switch (STR(n)[0]) {
case '+':
return Add;
case '-':
return Sub;
case '/':
if (STR(n)[1] == '/')
return FloorDiv;
else
return Div;
case '%':
return Mod;
case '<':
return LShift;
case '>':
return RShift;
case '&':
return BitAnd;
case '^':
return BitXor;
case '|':
return BitOr;
case '*':
if (STR(n)[1] == '*')
return Pow;
else
return Mult;
case '@':
if (c->c_feature_version < 5) {
ast_error(c, n,
"The '@' operator is only supported in Python 3.5 and greater");
return (operator_ty)0;
}
return MatMult;
default:
PyErr_Format(PyExc_SystemError, "invalid augassign: %s", STR(n));
return (operator_ty)0;
}
}
static cmpop_ty
ast_for_comp_op(struct compiling *c, const node *n)
{
/* comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'
|'is' 'not'
*/
REQ(n, comp_op);
if (NCH(n) == 1) {
n = CHILD(n, 0);
switch (TYPE(n)) {
case LESS:
return Lt;
case GREATER:
return Gt;
case EQEQUAL: /* == */
return Eq;
case LESSEQUAL:
return LtE;
case GREATEREQUAL:
return GtE;
case NOTEQUAL:
return NotEq;
case NAME:
if (strcmp(STR(n), "in") == 0)
return In;
if (strcmp(STR(n), "is") == 0)
return Is;
/* fall through */
default:
PyErr_Format(PyExc_SystemError, "invalid comp_op: %s",
STR(n));
return (cmpop_ty)0;
}
}
else if (NCH(n) == 2) {
/* handle "not in" and "is not" */
switch (TYPE(CHILD(n, 0))) {
case NAME:
if (strcmp(STR(CHILD(n, 1)), "in") == 0)
return NotIn;
if (strcmp(STR(CHILD(n, 0)), "is") == 0)
return IsNot;
/* fall through */
default:
PyErr_Format(PyExc_SystemError, "invalid comp_op: %s %s",
STR(CHILD(n, 0)), STR(CHILD(n, 1)));
return (cmpop_ty)0;
}
}
PyErr_Format(PyExc_SystemError, "invalid comp_op: has %d children",
NCH(n));
return (cmpop_ty)0;
}
static asdl_seq *
seq_for_testlist(struct compiling *c, const node *n)
{
/* testlist: test (',' test)* [',']
testlist_star_expr: test|star_expr (',' test|star_expr)* [',']
*/
asdl_seq *seq;
expr_ty expression;
int i;
assert(TYPE(n) == testlist || TYPE(n) == testlist_star_expr || TYPE(n) == testlist_comp);
seq = _Py_asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
if (!seq)
return NULL;
for (i = 0; i < NCH(n); i += 2) {
const node *ch = CHILD(n, i);
assert(TYPE(ch) == test || TYPE(ch) == test_nocond || TYPE(ch) == star_expr || TYPE(ch) == namedexpr_test);
expression = ast_for_expr(c, ch);
if (!expression)
return NULL;
assert(i / 2 < seq->size);
asdl_seq_SET(seq, i / 2, expression);
}
return seq;
}
static arg_ty
ast_for_arg(struct compiling *c, const node *n)
{
identifier name;
expr_ty annotation = NULL;
node *ch;
arg_ty ret;
assert(TYPE(n) == tfpdef || TYPE(n) == vfpdef);
ch = CHILD(n, 0);
name = NEW_IDENTIFIER(ch);
if (!name)
return NULL;
if (forbidden_name(c, name, ch, 0))
return NULL;
if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == COLON) {
annotation = ast_for_expr(c, CHILD(n, 2));
if (!annotation)
return NULL;
}
ret = arg(name, annotation, NULL, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (!ret)
return NULL;
return ret;
}
/* returns -1 if failed to handle keyword only arguments
returns new position to keep processing if successful
(',' tfpdef ['=' test])*
^^^
start pointing here
*/
static int
handle_keywordonly_args(struct compiling *c, const node *n, int start,
asdl_seq *kwonlyargs, asdl_seq *kwdefaults)
{
PyObject *argname;
node *ch;
expr_ty expression, annotation;
arg_ty arg = NULL;
int i = start;
int j = 0; /* index for kwdefaults and kwonlyargs */
if (kwonlyargs == NULL) {
ast_error(c, CHILD(n, start), "named arguments must follow bare *");
return -1;
}
assert(kwdefaults != NULL);
while (i < NCH(n)) {
ch = CHILD(n, i);
switch (TYPE(ch)) {
case vfpdef:
case tfpdef:
if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
expression = ast_for_expr(c, CHILD(n, i + 2));
if (!expression)
goto error;
asdl_seq_SET(kwdefaults, j, expression);
i += 2; /* '=' and test */
}
else { /* setting NULL if no default value exists */
asdl_seq_SET(kwdefaults, j, NULL);
}
if (NCH(ch) == 3) {
/* ch is NAME ':' test */
annotation = ast_for_expr(c, CHILD(ch, 2));
if (!annotation)
goto error;
}
else {
annotation = NULL;
}
ch = CHILD(ch, 0);
argname = NEW_IDENTIFIER(ch);
if (!argname)
goto error;
if (forbidden_name(c, argname, ch, 0))
goto error;
arg = arg(argname, annotation, NULL, LINENO(ch), ch->n_col_offset,
ch->n_end_lineno, ch->n_end_col_offset,
c->c_arena);
if (!arg)
goto error;
asdl_seq_SET(kwonlyargs, j++, arg);
i += 1; /* the name */
if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA)
i += 1; /* the comma, if present */
break;
case TYPE_COMMENT:
/* arg will be equal to the last argument processed */
arg->type_comment = NEW_TYPE_COMMENT(ch);
if (!arg->type_comment)
goto error;
i += 1;
break;
case DOUBLESTAR:
return i;
default:
ast_error(c, ch, "unexpected node");
goto error;
}
}
return i;
error:
return -1;
}
/* Create AST for argument list. */
static arguments_ty
ast_for_arguments(struct compiling *c, const node *n)
{
/* This function handles both typedargslist (function definition)
and varargslist (lambda definition).
parameters: '(' [typedargslist] ')'
The following definition for typedarglist is equivalent to this set of rules:
arguments = argument (',' [TYPE_COMMENT] argument)*
argument = tfpdef ['=' test]
kwargs = '**' tfpdef [','] [TYPE_COMMENT]
args = '*' [tfpdef]
kwonly_kwargs = (',' [TYPE_COMMENT] argument)* (TYPE_COMMENT | [','
[TYPE_COMMENT] [kwargs]])
args_kwonly_kwargs = args kwonly_kwargs | kwargs
poskeyword_args_kwonly_kwargs = arguments ( TYPE_COMMENT | [','
[TYPE_COMMENT] [args_kwonly_kwargs]])
typedargslist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs
typedarglist = (arguments ',' [TYPE_COMMENT] '/' [',' [[TYPE_COMMENT]
typedargslist_no_posonly]])|(typedargslist_no_posonly)"
typedargslist: ( (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])*
',' [TYPE_COMMENT] '/' [',' [ [TYPE_COMMENT] tfpdef ['=' test] ( ','
[TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [ '*'
[tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [','
[TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]]) | '**' tfpdef [',']
[TYPE_COMMENT]]]) | '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])*
(TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]]) |
'**' tfpdef [','] [TYPE_COMMENT]]] ) | (tfpdef ['=' test] (','
[TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [ '*'
[tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [','
[TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]]) | '**' tfpdef [',']
[TYPE_COMMENT]]]) | '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])*
(TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]]) |
'**' tfpdef [','] [TYPE_COMMENT]))
tfpdef: NAME [':' test]
The following definition for varargslist is equivalent to this set of rules:
arguments = argument (',' argument )*
argument = vfpdef ['=' test]
kwargs = '**' vfpdef [',']
args = '*' [vfpdef]
kwonly_kwargs = (',' argument )* [',' [kwargs]]
args_kwonly_kwargs = args kwonly_kwargs | kwargs
poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]]
vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs
varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] |
(vararglist_no_posonly)
varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef ['='
test] (',' vfpdef ['=' test])* [',' [ '*' [vfpdef] (',' vfpdef ['=' test])* [','
['**' vfpdef [',']]] | '**' vfpdef [',']]] | '*' [vfpdef] (',' vfpdef ['=' test])*
[',' ['**' vfpdef [',']]] | '**' vfpdef [',']) ]] | (vfpdef ['=' test] (',' vfpdef
['=' test])* [',' [ '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
| '**' vfpdef [',']]] | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef
[',']]] | '**' vfpdef [','])
vfpdef: NAME
*/
int i, j, k, l, nposonlyargs=0, nposargs = 0, nkwonlyargs = 0;
int nposdefaults = 0, found_default = 0;
asdl_seq *posonlyargs, *posargs, *posdefaults, *kwonlyargs, *kwdefaults;
arg_ty vararg = NULL, kwarg = NULL;
arg_ty arg = NULL;
node *ch;
if (TYPE(n) == parameters) {
if (NCH(n) == 2) /* () as argument list */
return arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL, c->c_arena);
n = CHILD(n, 1);
}
assert(TYPE(n) == typedargslist || TYPE(n) == varargslist);
/* First count the number of positional args & defaults. The
variable i is the loop index for this for loop and the next.
The next loop picks up where the first leaves off.
*/
for (i = 0; i < NCH(n); i++) {
ch = CHILD(n, i);
if (TYPE(ch) == STAR) {
/* skip star */
i++;
if (i < NCH(n) && /* skip argument following star */
(TYPE(CHILD(n, i)) == tfpdef ||
TYPE(CHILD(n, i)) == vfpdef)) {
i++;
}
break;
}
if (TYPE(ch) == DOUBLESTAR) break;
if (TYPE(ch) == vfpdef || TYPE(ch) == tfpdef) nposargs++;
if (TYPE(ch) == EQUAL) nposdefaults++;
if (TYPE(ch) == SLASH ) {
nposonlyargs = nposargs;
nposargs = 0;
}
}
/* count the number of keyword only args &
defaults for keyword only args */
for ( ; i < NCH(n); ++i) {
ch = CHILD(n, i);
if (TYPE(ch) == DOUBLESTAR) break;
if (TYPE(ch) == tfpdef || TYPE(ch) == vfpdef) nkwonlyargs++;
}
posonlyargs = (nposonlyargs ? _Py_asdl_seq_new(nposonlyargs, c->c_arena) : NULL);
if (!posonlyargs && nposonlyargs) {
return NULL;
}
posargs = (nposargs ? _Py_asdl_seq_new(nposargs, c->c_arena) : NULL);
if (!posargs && nposargs)
return NULL;
kwonlyargs = (nkwonlyargs ?
_Py_asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
if (!kwonlyargs && nkwonlyargs)
return NULL;
posdefaults = (nposdefaults ?
_Py_asdl_seq_new(nposdefaults, c->c_arena) : NULL);
if (!posdefaults && nposdefaults)
return NULL;
/* The length of kwonlyargs and kwdefaults are same
since we set NULL as default for keyword only argument w/o default
- we have sequence data structure, but no dictionary */
kwdefaults = (nkwonlyargs ?
_Py_asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
if (!kwdefaults && nkwonlyargs)
return NULL;
/* tfpdef: NAME [':' test]
vfpdef: NAME
*/
i = 0;
j = 0; /* index for defaults */
k = 0; /* index for args */
l = 0; /* index for posonlyargs */
while (i < NCH(n)) {
ch = CHILD(n, i);
switch (TYPE(ch)) {
case tfpdef:
case vfpdef:
/* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is
anything other than EQUAL or a comma? */
/* XXX Should NCH(n) check be made a separate check? */
if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
expr_ty expression = ast_for_expr(c, CHILD(n, i + 2));
if (!expression)
return NULL;
assert(posdefaults != NULL);
asdl_seq_SET(posdefaults, j++, expression);
i += 2;
found_default = 1;
}
else if (found_default) {
ast_error(c, n,
"non-default argument follows default argument");
return NULL;
}
arg = ast_for_arg(c, ch);
if (!arg)
return NULL;
if (l < nposonlyargs) {
asdl_seq_SET(posonlyargs, l++, arg);
} else {
asdl_seq_SET(posargs, k++, arg);
}
i += 1; /* the name */
if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA)
i += 1; /* the comma, if present */
break;
case SLASH:
/* Advance the slash and the comma. If there are more names
* after the slash there will be a comma so we are advancing
* the correct number of nodes. If the slash is the last item,
* we will be advancing an extra token but then * i > NCH(n)
* and the enclosing while will finish correctly. */
i += 2;
break;
case STAR:
if (i+1 >= NCH(n) ||
(i+2 == NCH(n) && (TYPE(CHILD(n, i+1)) == COMMA
|| TYPE(CHILD(n, i+1)) == TYPE_COMMENT))) {
ast_error(c, CHILD(n, i),
"named arguments must follow bare *");
return NULL;
}
ch = CHILD(n, i+1); /* tfpdef or COMMA */
if (TYPE(ch) == COMMA) {
int res = 0;
i += 2; /* now follows keyword only arguments */
if (i < NCH(n) && TYPE(CHILD(n, i)) == TYPE_COMMENT) {
ast_error(c, CHILD(n, i),
"bare * has associated type comment");
return NULL;
}
res = handle_keywordonly_args(c, n, i,
kwonlyargs, kwdefaults);
if (res == -1) return NULL;
i = res; /* res has new position to process */
}
else {
vararg = ast_for_arg(c, ch);
if (!vararg)
return NULL;
i += 2; /* the star and the name */
if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA)
i += 1; /* the comma, if present */
if (i < NCH(n) && TYPE(CHILD(n, i)) == TYPE_COMMENT) {
vararg->type_comment = NEW_TYPE_COMMENT(CHILD(n, i));
if (!vararg->type_comment)
return NULL;
i += 1;
}
if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef
|| TYPE(CHILD(n, i)) == vfpdef)) {
int res = 0;
res = handle_keywordonly_args(c, n, i,
kwonlyargs, kwdefaults);
if (res == -1) return NULL;
i = res; /* res has new position to process */
}
}
break;
case DOUBLESTAR:
ch = CHILD(n, i+1); /* tfpdef */
assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef);
kwarg = ast_for_arg(c, ch);
if (!kwarg)
return NULL;
i += 2; /* the double star and the name */
if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA)
i += 1; /* the comma, if present */
break;
case TYPE_COMMENT:
assert(i);
if (kwarg)
arg = kwarg;
/* arg will be equal to the last argument processed */
arg->type_comment = NEW_TYPE_COMMENT(ch);
if (!arg->type_comment)
return NULL;
i += 1;
break;
default:
PyErr_Format(PyExc_SystemError,
"unexpected node in varargslist: %d @ %d",
TYPE(ch), i);
return NULL;
}
}
return arguments(posonlyargs, posargs, vararg, kwonlyargs, kwdefaults, kwarg, posdefaults, c->c_arena);
}
static expr_ty
ast_for_dotted_name(struct compiling *c, const node *n)
{
expr_ty e;
identifier id;
int lineno, col_offset;
int i;
node *ch;
REQ(n, dotted_name);
lineno = LINENO(n);
col_offset = n->n_col_offset;
ch = CHILD(n, 0);
id = NEW_IDENTIFIER(ch);
if (!id)
return NULL;
e = Name(id, Load, lineno, col_offset,
ch->n_end_lineno, ch->n_end_col_offset, c->c_arena);
if (!e)
return NULL;
for (i = 2; i < NCH(n); i+=2) {
const node *child = CHILD(n, i);
id = NEW_IDENTIFIER(child);
if (!id)
return NULL;
e = Attribute(e, id, Load, lineno, col_offset,
child->n_end_lineno, child->n_end_col_offset, c->c_arena);
if (!e)
return NULL;
}
return e;
}
static expr_ty
ast_for_decorator(struct compiling *c, const node *n)
{
/* decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE */
expr_ty d = NULL;
expr_ty name_expr;
REQ(n, decorator);
REQ(CHILD(n, 0), AT);
REQ(RCHILD(n, -1), NEWLINE);
name_expr = ast_for_dotted_name(c, CHILD(n, 1));
if (!name_expr)
return NULL;
if (NCH(n) == 3) { /* No arguments */
d = name_expr;
name_expr = NULL;
}
else if (NCH(n) == 5) { /* Call with no arguments */
d = Call(name_expr, NULL, NULL,
name_expr->lineno, name_expr->col_offset,
CHILD(n, 3)->n_end_lineno, CHILD(n, 3)->n_end_col_offset,
c->c_arena);
if (!d)
return NULL;
name_expr = NULL;
}
else {
d = ast_for_call(c, CHILD(n, 3), name_expr,
CHILD(n, 1), CHILD(n, 2), CHILD(n, 4));
if (!d)
return NULL;
name_expr = NULL;
}
return d;
}
static asdl_seq*
ast_for_decorators(struct compiling *c, const node *n)
{
asdl_seq* decorator_seq;
expr_ty d;
int i;
REQ(n, decorators);
decorator_seq = _Py_asdl_seq_new(NCH(n), c->c_arena);
if (!decorator_seq)
return NULL;
for (i = 0; i < NCH(n); i++) {
d = ast_for_decorator(c, CHILD(n, i));
if (!d)
return NULL;
asdl_seq_SET(decorator_seq, i, d);
}
return decorator_seq;
}
static stmt_ty
ast_for_funcdef_impl(struct compiling *c, const node *n0,
asdl_seq *decorator_seq, bool is_async)
{
/* funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] suite */
const node * const n = is_async ? CHILD(n0, 1) : n0;
identifier name;
arguments_ty args;
asdl_seq *body;
expr_ty returns = NULL;
int name_i = 1;
int end_lineno, end_col_offset;
node *tc;
string type_comment = NULL;
if (is_async && c->c_feature_version < 5) {
ast_error(c, n,
"Async functions are only supported in Python 3.5 and greater");
return NULL;
}
REQ(n, funcdef);
name = NEW_IDENTIFIER(CHILD(n, name_i));
if (!name)
return NULL;
if (forbidden_name(c, name, CHILD(n, name_i), 0))
return NULL;
args = ast_for_arguments(c, CHILD(n, name_i + 1));
if (!args)
return NULL;
if (TYPE(CHILD(n, name_i+2)) == RARROW) {
returns = ast_for_expr(c, CHILD(n, name_i + 3));
if (!returns)
return NULL;
name_i += 2;
}
if (TYPE(CHILD(n, name_i + 3)) == TYPE_COMMENT) {
type_comment = NEW_TYPE_COMMENT(CHILD(n, name_i + 3));
if (!type_comment)
return NULL;
name_i += 1;
}
body = ast_for_suite(c, CHILD(n, name_i + 3));
if (!body)
return NULL;
get_last_end_pos(body, &end_lineno, &end_col_offset);
if (NCH(CHILD(n, name_i + 3)) > 1) {
/* Check if the suite has a type comment in it. */
tc = CHILD(CHILD(n, name_i + 3), 1);
if (TYPE(tc) == TYPE_COMMENT) {
if (type_comment != NULL) {
ast_error(c, n, "Cannot have two type comments on def");
return NULL;
}
type_comment = NEW_TYPE_COMMENT(tc);
if (!type_comment)
return NULL;
}
}
if (is_async)
return AsyncFunctionDef(name, args, body, decorator_seq, returns, type_comment,
LINENO(n0), n0->n_col_offset, end_lineno, end_col_offset, c->c_arena);
else
return FunctionDef(name, args, body, decorator_seq, returns, type_comment,
LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena);
}
static stmt_ty
ast_for_async_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
/* async_funcdef: ASYNC funcdef */
REQ(n, async_funcdef);
REQ(CHILD(n, 0), ASYNC);
REQ(CHILD(n, 1), funcdef);
return ast_for_funcdef_impl(c, n, decorator_seq,
true /* is_async */);
}
static stmt_ty
ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
/* funcdef: 'def' NAME parameters ['->' test] ':' suite */
return ast_for_funcdef_impl(c, n, decorator_seq,
false /* is_async */);
}
static stmt_ty
ast_for_async_stmt(struct compiling *c, const node *n)
{
/* async_stmt: ASYNC (funcdef | with_stmt | for_stmt) */
REQ(n, async_stmt);
REQ(CHILD(n, 0), ASYNC);
switch (TYPE(CHILD(n, 1))) {
case funcdef:
return ast_for_funcdef_impl(c, n, NULL,
true /* is_async */);
case with_stmt:
return ast_for_with_stmt(c, n,
true /* is_async */);
case for_stmt:
return ast_for_for_stmt(c, n,
true /* is_async */);
default:
PyErr_Format(PyExc_SystemError,
"invalid async stament: %s",
STR(CHILD(n, 1)));
return NULL;
}
}
static stmt_ty
ast_for_decorated(struct compiling *c, const node *n)
{
/* decorated: decorators (classdef | funcdef | async_funcdef) */
stmt_ty thing = NULL;
asdl_seq *decorator_seq = NULL;
REQ(n, decorated);
decorator_seq = ast_for_decorators(c, CHILD(n, 0));
if (!decorator_seq)
return NULL;
assert(TYPE(CHILD(n, 1)) == funcdef ||
TYPE(CHILD(n, 1)) == async_funcdef ||
TYPE(CHILD(n, 1)) == classdef);
if (TYPE(CHILD(n, 1)) == funcdef) {
thing = ast_for_funcdef(c, CHILD(n, 1), decorator_seq);
} else if (TYPE(CHILD(n, 1)) == classdef) {
thing = ast_for_classdef(c, CHILD(n, 1), decorator_seq);
} else if (TYPE(CHILD(n, 1)) == async_funcdef) {
thing = ast_for_async_funcdef(c, CHILD(n, 1), decorator_seq);
}
return thing;
}
static expr_ty
ast_for_namedexpr(struct compiling *c, const node *n)
{
/* namedexpr_test: test [':=' test]
argument: ( test [comp_for] |
test ':=' test |
test '=' test |
'**' test |
'*' test )
*/
expr_ty target, value;
target = ast_for_expr(c, CHILD(n, 0));
if (!target)
return NULL;
value = ast_for_expr(c, CHILD(n, 2));
if (!value)
return NULL;
if (target->kind != Name_kind) {
const char *expr_name = get_expr_name(target);
if (expr_name != NULL) {
ast_error(c, n, "cannot use assignment expressions with %s", expr_name);
}
return NULL;
}
if (!set_context(c, target, Store, n))
return NULL;
return NamedExpr(target, value, LINENO(n), n->n_col_offset, n->n_end_lineno,
n->n_end_col_offset, c->c_arena);
}
static expr_ty
ast_for_lambdef(struct compiling *c, const node *n)
{
/* lambdef: 'lambda' [varargslist] ':' test
lambdef_nocond: 'lambda' [varargslist] ':' test_nocond */
arguments_ty args;
expr_ty expression;
if (NCH(n) == 3) {
args = arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL, c->c_arena);
if (!args)
return NULL;
expression = ast_for_expr(c, CHILD(n, 2));
if (!expression)
return NULL;
}
else {
args = ast_for_arguments(c, CHILD(n, 1));
if (!args)
return NULL;
expression = ast_for_expr(c, CHILD(n, 3));
if (!expression)
return NULL;
}
return Lambda(args, expression, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static expr_ty
ast_for_ifexpr(struct compiling *c, const node *n)
{
/* test: or_test 'if' or_test 'else' test */
expr_ty expression, body, orelse;
assert(NCH(n) == 5);
body = ast_for_expr(c, CHILD(n, 0));
if (!body)
return NULL;
expression = ast_for_expr(c, CHILD(n, 2));
if (!expression)
return NULL;
orelse = ast_for_expr(c, CHILD(n, 4));
if (!orelse)
return NULL;
return IfExp(expression, body, orelse, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset,
c->c_arena);
}
/*
Count the number of 'for' loops in a comprehension.
Helper for ast_for_comprehension().
*/
static int
count_comp_fors(struct compiling *c, const node *n)
{
int n_fors = 0;
count_comp_for:
n_fors++;
REQ(n, comp_for);
if (NCH(n) == 2) {
REQ(CHILD(n, 0), ASYNC);
n = CHILD(n, 1);
}
else if (NCH(n) == 1) {
n = CHILD(n, 0);
}
else {
goto error;
}
if (NCH(n) == (5)) {
n = CHILD(n, 4);
}
else {
return n_fors;
}
count_comp_iter:
REQ(n, comp_iter);
n = CHILD(n, 0);
if (TYPE(n) == comp_for)
goto count_comp_for;
else if (TYPE(n) == comp_if) {
if (NCH(n) == 3) {
n = CHILD(n, 2);
goto count_comp_iter;
}
else
return n_fors;
}
error:
/* Should never be reached */
PyErr_SetString(PyExc_SystemError,
"logic error in count_comp_fors");
return -1;
}
/* Count the number of 'if' statements in a comprehension.
Helper for ast_for_comprehension().
*/
static int
count_comp_ifs(struct compiling *c, const node *n)
{
int n_ifs = 0;
while (1) {
REQ(n, comp_iter);
if (TYPE(CHILD(n, 0)) == comp_for)
return n_ifs;
n = CHILD(n, 0);
REQ(n, comp_if);
n_ifs++;
if (NCH(n) == 2)
return n_ifs;
n = CHILD(n, 2);
}
}
static asdl_seq *
ast_for_comprehension(struct compiling *c, const node *n)
{
int i, n_fors;
asdl_seq *comps;
n_fors = count_comp_fors(c, n);
if (n_fors == -1)
return NULL;
comps = _Py_asdl_seq_new(n_fors, c->c_arena);
if (!comps)
return NULL;
for (i = 0; i < n_fors; i++) {
comprehension_ty comp;
asdl_seq *t;
expr_ty expression, first;
node *for_ch;
node *sync_n;
int is_async = 0;
REQ(n, comp_for);
if (NCH(n) == 2) {
is_async = 1;
REQ(CHILD(n, 0), ASYNC);
sync_n = CHILD(n, 1);
}
else {
sync_n = CHILD(n, 0);
}
REQ(sync_n, sync_comp_for);
/* Async comprehensions only allowed in Python 3.6 and greater */
if (is_async && c->c_feature_version < 6) {
ast_error(c, n,
"Async comprehensions are only supported in Python 3.6 and greater");
return NULL;
}
for_ch = CHILD(sync_n, 1);
t = ast_for_exprlist(c, for_ch, Store);
if (!t)
return NULL;
expression = ast_for_expr(c, CHILD(sync_n, 3));
if (!expression)
return NULL;
/* Check the # of children rather than the length of t, since
(x for x, in ...) has 1 element in t, but still requires a Tuple. */
first = (expr_ty)asdl_seq_GET(t, 0);
if (NCH(for_ch) == 1)
comp = comprehension(first, expression, NULL,
is_async, c->c_arena);
else
comp = comprehension(Tuple(t, Store, first->lineno, first->col_offset,
for_ch->n_end_lineno, for_ch->n_end_col_offset,
c->c_arena),
expression, NULL, is_async, c->c_arena);
if (!comp)
return NULL;
if (NCH(sync_n) == 5) {
int j, n_ifs;
asdl_seq *ifs;
n = CHILD(sync_n, 4);
n_ifs = count_comp_ifs(c, n);
if (n_ifs == -1)
return NULL;
ifs = _Py_asdl_seq_new(n_ifs, c->c_arena);
if (!ifs)
return NULL;
for (j = 0; j < n_ifs; j++) {
REQ(n, comp_iter);
n = CHILD(n, 0);
REQ(n, comp_if);
expression = ast_for_expr(c, CHILD(n, 1));
if (!expression)
return NULL;
asdl_seq_SET(ifs, j, expression);
if (NCH(n) == 3)
n = CHILD(n, 2);
}
/* on exit, must guarantee that n is a comp_for */
if (TYPE(n) == comp_iter)
n = CHILD(n, 0);
comp->ifs = ifs;
}
asdl_seq_SET(comps, i, comp);
}
return comps;
}
static expr_ty
ast_for_itercomp(struct compiling *c, const node *n, int type)
{
/* testlist_comp: (test|star_expr)
* ( comp_for | (',' (test|star_expr))* [','] ) */
expr_ty elt;
asdl_seq *comps;
node *ch;
assert(NCH(n) > 1);
ch = CHILD(n, 0);
elt = ast_for_expr(c, ch);
if (!elt)
return NULL;
if (elt->kind == Starred_kind) {
ast_error(c, ch, "iterable unpacking cannot be used in comprehension");
return NULL;
}
comps = ast_for_comprehension(c, CHILD(n, 1));
if (!comps)
return NULL;
if (type == COMP_GENEXP)
return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
else if (type == COMP_LISTCOMP)
return ListComp(elt, comps, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
else if (type == COMP_SETCOMP)
return SetComp(elt, comps, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
else
/* Should never happen */
return NULL;
}
/* Fills in the key, value pair corresponding to the dict element. In case
* of an unpacking, key is NULL. *i is advanced by the number of ast
* elements. Iff successful, nonzero is returned.
*/
static int
ast_for_dictelement(struct compiling *c, const node *n, int *i,
expr_ty *key, expr_ty *value)
{
expr_ty expression;
if (TYPE(CHILD(n, *i)) == DOUBLESTAR) {
assert(NCH(n) - *i >= 2);
expression = ast_for_expr(c, CHILD(n, *i + 1));
if (!expression)
return 0;
*key = NULL;
*value = expression;
*i += 2;
}
else {
assert(NCH(n) - *i >= 3);
expression = ast_for_expr(c, CHILD(n, *i));
if (!expression)
return 0;
*key = expression;
REQ(CHILD(n, *i + 1), COLON);
expression = ast_for_expr(c, CHILD(n, *i + 2));
if (!expression)
return 0;
*value = expression;
*i += 3;
}
return 1;
}
static expr_ty
ast_for_dictcomp(struct compiling *c, const node *n)
{
expr_ty key, value;
asdl_seq *comps;
int i = 0;
if (!ast_for_dictelement(c, n, &i, &key, &value))
return NULL;
assert(key);
assert(NCH(n) - i >= 1);
comps = ast_for_comprehension(c, CHILD(n, i));
if (!comps)
return NULL;
return DictComp(key, value, comps, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static expr_ty
ast_for_dictdisplay(struct compiling *c, const node *n)
{
int i;
int j;
int size;
asdl_seq *keys, *values;
size = (NCH(n) + 1) / 3; /* +1 in case no trailing comma */
keys = _Py_asdl_seq_new(size, c->c_arena);
if (!keys)
return NULL;
values = _Py_asdl_seq_new(size, c->c_arena);
if (!values)
return NULL;
j = 0;
for (i = 0; i < NCH(n); i++) {
expr_ty key, value;
if (!ast_for_dictelement(c, n, &i, &key, &value))
return NULL;
asdl_seq_SET(keys, j, key);
asdl_seq_SET(values, j, value);
j++;
}
keys->size = j;
values->size = j;
return Dict(keys, values, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static expr_ty
ast_for_genexp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (testlist_comp) || TYPE(n) == (argument));
return ast_for_itercomp(c, n, COMP_GENEXP);
}
static expr_ty
ast_for_listcomp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (testlist_comp));
return ast_for_itercomp(c, n, COMP_LISTCOMP);
}
static expr_ty
ast_for_setcomp(struct compiling *c, const node *n)
{
assert(TYPE(n) == (dictorsetmaker));
return ast_for_itercomp(c, n, COMP_SETCOMP);
}
static expr_ty
ast_for_setdisplay(struct compiling *c, const node *n)
{
int i;
int size;
asdl_seq *elts;
assert(TYPE(n) == (dictorsetmaker));
size = (NCH(n) + 1) / 2; /* +1 in case no trailing comma */
elts = _Py_asdl_seq_new(size, c->c_arena);
if (!elts)
return NULL;
for (i = 0; i < NCH(n); i += 2) {
expr_ty expression;
expression = ast_for_expr(c, CHILD(n, i));
if (!expression)
return NULL;
asdl_seq_SET(elts, i / 2, expression);
}
return Set(elts, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
static expr_ty
ast_for_atom(struct compiling *c, const node *n)
{
/* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']'
| '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+
| '...' | 'None' | 'True' | 'False'
*/
node *ch = CHILD(n, 0);
switch (TYPE(ch)) {
case NAME: {
PyObject *name;
const char *s = STR(ch);
size_t len = strlen(s);
if (len >= 4 && len <= 5) {
if (!strcmp(s, "None"))
return Constant(Py_None, NULL, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (!strcmp(s, "True"))
return Constant(Py_True, NULL, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (!strcmp(s, "False"))
return Constant(Py_False, NULL, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
name = new_identifier(s, c);
if (!name)
return NULL;
/* All names start in Load context, but may later be changed. */
return Name(name, Load, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
case STRING: {
expr_ty str = parsestrplus(c, n);
if (!str) {
const char *errtype = NULL;
if (PyErr_ExceptionMatches(PyExc_UnicodeError))
errtype = "unicode error";
else if (PyErr_ExceptionMatches(PyExc_ValueError))
errtype = "value error";
if (errtype) {
PyObject *type, *value, *tback, *errstr;
PyErr_Fetch(&type, &value, &tback);
errstr = PyObject_Str(value);
if (errstr) {
ast_error(c, n, "(%s) %U", errtype, errstr);
Py_DECREF(errstr);
}
else {
PyErr_Clear();
ast_error(c, n, "(%s) unknown error", errtype);
}
Py_DECREF(type);
Py_XDECREF(value);
Py_XDECREF(tback);
}
return NULL;
}
return str;
}
case NUMBER: {
PyObject *pynum;
/* Underscores in numeric literals are only allowed in Python 3.6 or greater */
/* Check for underscores here rather than in parse_number so we can report a line number on error */
if (c->c_feature_version < 6 && strchr(STR(ch), '_') != NULL) {
ast_error(c, ch,
"Underscores in numeric literals are only supported in Python 3.6 and greater");
return NULL;
}
pynum = parsenumber(c, STR(ch));
if (!pynum) {
PyThreadState *tstate = PyThreadState_GET();
// The only way a ValueError should happen in _this_ code is via
// PyLong_FromString hitting a length limit.
if (tstate->curexc_type == PyExc_ValueError &&
tstate->curexc_value != NULL) {
PyObject *type, *value, *tb;
// This acts as PyErr_Clear() as we're replacing curexc.
PyErr_Fetch(&type, &value, &tb);
Py_XDECREF(tb);
Py_DECREF(type);
ast_error(c, ch,
"%S - Consider hexadecimal for huge integer literals "
"to avoid decimal conversion limits.",
value);
Py_DECREF(value);
}
return NULL;
}
if (PyArena_AddPyObject(c->c_arena, pynum) < 0) {
Py_DECREF(pynum);
return NULL;
}
return Constant(pynum, NULL, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
case ELLIPSIS: /* Ellipsis */
return Constant(Py_Ellipsis, NULL, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
case LPAR: /* some parenthesized expressions */
ch = CHILD(n, 1);
if (TYPE(ch) == RPAR)
return Tuple(NULL, Load, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
if (TYPE(ch) == yield_expr)
return ast_for_expr(c, ch);
/* testlist_comp: test ( comp_for | (',' test)* [','] ) */
if (NCH(ch) == 1) {
return ast_for_testlist(c, ch);
}
if (TYPE(CHILD(ch, 1)) == comp_for) {
return copy_location(ast_for_genexp(c, ch), n, n);
}
else {
return copy_location(ast_for_testlist(c, ch), n, n);
}
case LSQB: /* list (or list comprehension) */
ch = CHILD(n, 1);
if (TYPE(ch) == RSQB)
return List(NULL, Load, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
REQ(ch, testlist_comp);
if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
asdl_seq *elts = seq_for_testlist(c, ch);
if (!elts)
return NULL;
return List(elts, Load, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else {
return copy_location(ast_for_listcomp(c, ch), n, n);
}
case LBRACE: {
/* dictorsetmaker: ( ((test ':' test | '**' test)
* (comp_for | (',' (test ':' test | '**' test))* [','])) |
* ((test | '*' test)
* (comp_for | (',' (test | '*' test))* [','])) ) */
expr_ty res;
ch = CHILD(n, 1);
if (TYPE(ch) == RBRACE) {
/* It's an empty dict. */
return Dict(NULL, NULL, LINENO(n), n->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else {
int is_dict = (TYPE(CHILD(ch, 0)) == DOUBLESTAR);
if (NCH(ch) == 1 ||
(NCH(ch) > 1 &&
TYPE(CHILD(ch, 1)) == COMMA)) {
/* It's a set display. */
res = ast_for_setdisplay(c, ch);
}
else if (NCH(ch) > 1 &&
TYPE(CHILD(ch, 1)) == comp_for) {
/* It's a set comprehension. */
res = ast_for_setcomp(c, ch);
}
else if (NCH(ch) > 3 - is_dict &&
TYPE(CHILD(ch, 3 - is_dict)) == comp_for) {
/* It's a dictionary comprehension. */
if (is_dict) {
ast_error(c, n,
"dict unpacking cannot be used in dict comprehension");
return NULL;
}
res = ast_for_dictcomp(c, ch);
}
else {
/* It's a dictionary display. */
res = ast_for_dictdisplay(c, ch);
}
return copy_location(res, n, n);
}
}
default:
PyErr_Format(PyExc_SystemError, "unhandled atom %d", TYPE(ch));
return NULL;
}
}
static slice_ty
ast_for_slice(struct compiling *c, const node *n)
{
node *ch;
expr_ty lower = NULL, upper = NULL, step = NULL;
REQ(n, subscript);
/*
subscript: test | [test] ':' [test] [sliceop]
sliceop: ':' [test]
*/
ch = CHILD(n, 0);
if (NCH(n) == 1 && TYPE(ch) == test) {
/* 'step' variable hold no significance in terms of being used over
other vars */
step = ast_for_expr(c, ch);
if (!step)
return NULL;
return Index(step, c->c_arena);
}
if (TYPE(ch) == test) {
lower = ast_for_expr(c, ch);
if (!lower)
return NULL;
}
/* If there's an upper bound it's in the second or third position. */
if (TYPE(ch) == COLON) {
if (NCH(n) > 1) {
node *n2 = CHILD(n, 1);
if (TYPE(n2) == test) {
upper = ast_for_expr(c, n2);
if (!upper)
return NULL;
}
}
} else if (NCH(n) > 2) {
node *n2 = CHILD(n, 2);
if (TYPE(n2) == test) {
upper = ast_for_expr(c, n2);
if (!upper)
return NULL;
}
}
ch = CHILD(n, NCH(n) - 1);
if (TYPE(ch) == sliceop) {
if (NCH(ch) != 1) {
ch = CHILD(ch, 1);
if (TYPE(ch) == test) {
step = ast_for_expr(c, ch);
if (!step)
return NULL;
}
}
}
return Slice(lower, upper, step, c->c_arena);
}
static expr_ty
ast_for_binop(struct compiling *c, const node *n)
{
/* Must account for a sequence of expressions.
How should A op B op C by represented?
BinOp(BinOp(A, op, B), op, C).
*/
int i, nops;
expr_ty expr1, expr2, result;
operator_ty newoperator;
expr1 = ast_for_expr(c, CHILD(n, 0));
if (!expr1)
return NULL;
expr2 = ast_for_expr(c, CHILD(n, 2));
if (!expr2)
return NULL;
newoperator = get_operator(c, CHILD(n, 1));
if (!newoperator)
return NULL;
result = BinOp(expr1, newoperator, expr2, LINENO(n), n->n_col_offset,
CHILD(n, 2)->n_end_lineno, CHILD(n, 2)->n_end_col_offset,
c->c_arena);
if (!result)
return NULL;
nops = (NCH(n) - 1) / 2;
for (i = 1; i < nops; i++) {
expr_ty tmp_result, tmp;
const node* next_oper = CHILD(n, i * 2 + 1);
newoperator = get_operator(c, next_oper);
if (!newoperator)
return NULL;
tmp = ast_for_expr(c, CHILD(n, i * 2 + 2));
if (!tmp)
return NULL;
tmp_result = BinOp(result, newoperator, tmp,
LINENO(n), n->n_col_offset,
CHILD(n, i * 2 + 2)->n_end_lineno,
CHILD(n, i * 2 + 2)->n_end_col_offset,
c->c_arena);
if (!tmp_result)
return NULL;
result = tmp_result;
}
return result;
}
static expr_ty
ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr, const node *start)
{
/* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [',']
subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
*/
const node *n_copy = n;
REQ(n, trailer);
if (TYPE(CHILD(n, 0)) == LPAR) {
if (NCH(n) == 2)
return Call(left_expr, NULL, NULL, LINENO(start), start->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
else
return ast_for_call(c, CHILD(n, 1), left_expr,
start, CHILD(n, 0), CHILD(n, 2));
}
else if (TYPE(CHILD(n, 0)) == DOT) {
PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1));
if (!attr_id)
return NULL;
return Attribute(left_expr, attr_id, Load,
LINENO(start), start->n_col_offset,
n->n_end_lineno, n->n_end_col_offset, c->c_arena);
}
else {
REQ(CHILD(n, 0), LSQB);
REQ(CHILD(n, 2), RSQB);
n = CHILD(n, 1);
if (NCH(n) == 1) {
slice_ty slc = ast_for_slice(c, CHILD(n, 0));
if (!slc)
return NULL;
return Subscript(left_expr, slc, Load, LINENO(start), start->n_col_offset,
n_copy->n_end_lineno, n_copy->n_end_col_offset,
c->c_arena);
}
else {
/* The grammar is ambiguous here. The ambiguity is resolved
by treating the sequence as a tuple literal if there are
no slice features.
*/
Py_ssize_t j;
slice_ty slc;
expr_ty e;
int simple = 1;
asdl_seq *slices, *elts;
slices = _Py_asdl_seq_new((NCH(n) +