djsundog
/
circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

py: Improve handling of memory error in parser. 

Parser shouldn't raise exceptions, so needs to check when memory
allocation fails.  This patch does that for the initial set up of the
parser state.

Also, we now put the parser object on the stack.  It's small enough to
go there instead of on the heap.

This partially addresses issue #558.
This commit is contained in:
Damien George 2014-05-10 17:36:41 +01:00
parent ad6178bb08
commit 1b82e9af5c
2 changed files with 53 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
py/misc.h
View File

@ -50,6 +50,7 @@ typedef unsigned int uint;
// TODO make a lazy m_renew that can increase by a smaller amount than requested (but by at least 1 more element) // TODO make a lazy m_renew that can increase by a smaller amount than requested (but by at least 1 more element)
#define m_new(type, num) ((type*)(m_malloc(sizeof(type) * (num)))) #define m_new(type, num) ((type*)(m_malloc(sizeof(type) * (num))))
#define m_new_maybe(type, num) ((type*)(m_malloc_maybe(sizeof(type) * (num))))
#define m_new0(type, num) ((type*)(m_malloc0(sizeof(type) * (num)))) #define m_new0(type, num) ((type*)(m_malloc0(sizeof(type) * (num))))
#define m_new_obj(type) (m_new(type, 1)) #define m_new_obj(type) (m_new(type, 1))
#define m_new_obj_var(obj_type, var_type, var_num) ((obj_type*)m_malloc(sizeof(obj_type) + sizeof(var_type) * (var_num))) #define m_new_obj_var(obj_type, var_type, var_num) ((obj_type*)m_malloc(sizeof(obj_type) + sizeof(var_type) * (var_num)))

99
py/parse.c
View File

@ -344,21 +344,26 @@ STATIC void push_result_rule(parser_t *parser, int src_line, const rule_t *rule,
mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_parse_error_kind_t *parse_error_kind_out) { mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_parse_error_kind_t *parse_error_kind_out) {
// allocate memory for the parser and its stacks // initialise parser and allocate memory for its stacks
parser_t *parser = m_new_obj(parser_t); parser_t parser;
parser->had_memory_error = false; parser.had_memory_error = false;
parser->rule_stack_alloc = MP_ALLOC_PARSE_RULE_INIT; parser.rule_stack_alloc = MP_ALLOC_PARSE_RULE_INIT;
parser->rule_stack_top = 0; parser.rule_stack_top = 0;
parser->rule_stack = m_new(rule_stack_t, parser->rule_stack_alloc); parser.rule_stack = m_new_maybe(rule_stack_t, parser.rule_stack_alloc);
parser->result_stack_alloc = MP_ALLOC_PARSE_RESULT_INIT; parser.result_stack_alloc = MP_ALLOC_PARSE_RESULT_INIT;
parser->result_stack_top = 0; parser.result_stack_top = 0;
parser->result_stack = m_new(mp_parse_node_t, parser->result_stack_alloc); parser.result_stack = m_new_maybe(mp_parse_node_t, parser.result_stack_alloc);
parser->lexer = lex; parser.lexer = lex;
// check if we could allocate the stacks
if (parser.rule_stack == NULL || parser.result_stack == NULL) {
goto memory_error;
}
// work out the top-level rule to use, and push it on the stack // work out the top-level rule to use, and push it on the stack
int top_level_rule; int top_level_rule;
@ -367,7 +372,7 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break; case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
default: top_level_rule = RULE_file_input; default: top_level_rule = RULE_file_input;
} }
push_rule(parser, mp_lexer_cur(lex)->src_line, rules[top_level_rule], 0); push_rule(&parser, mp_lexer_cur(lex)->src_line, rules[top_level_rule], 0);
// parse! // parse!
@ -381,17 +386,17 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
for (;;) { for (;;) {
next_rule: next_rule:
if (parser->rule_stack_top == 0 || parser->had_memory_error) { if (parser.rule_stack_top == 0 || parser.had_memory_error) {
break; break;
} }
pop_rule(parser, &rule, &i, &rule_src_line); pop_rule(&parser, &rule, &i, &rule_src_line);
n = rule->act & RULE_ACT_ARG_MASK; n = rule->act & RULE_ACT_ARG_MASK;
/* /*
// debugging // debugging
printf("depth=%d ", parser->rule_stack_top); printf("depth=%d ", parser.rule_stack_top);
for (int j = 0; j < parser->rule_stack_top; ++j) { for (int j = 0; j < parser.rule_stack_top; ++j) {
printf(" "); printf(" ");
} }
printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack); printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack);
@ -408,14 +413,14 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
switch (rule->arg[i] & RULE_ARG_KIND_MASK) { switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
case RULE_ARG_TOK: case RULE_ARG_TOK:
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) { if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(parser, lex); push_result_token(&parser, lex);
mp_lexer_to_next(lex); mp_lexer_to_next(lex);
goto next_rule; goto next_rule;
} }
break; break;
case RULE_ARG_RULE: case RULE_ARG_RULE:
push_rule(parser, rule_src_line, rule, i + 1); // save this or-rule push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule
push_rule_from_arg(parser, rule->arg[i]); // push child of or-rule push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule
goto next_rule; goto next_rule;
default: default:
assert(0); assert(0);
@ -423,14 +428,14 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
} }
if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) { if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(parser, lex); push_result_token(&parser, lex);
mp_lexer_to_next(lex); mp_lexer_to_next(lex);
} else { } else {
backtrack = true; backtrack = true;
goto next_rule; goto next_rule;
} }
} else { } else {
push_rule_from_arg(parser, rule->arg[i]); push_rule_from_arg(&parser, rule->arg[i]);
} }
break; break;
@ -441,7 +446,7 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
assert(i > 0); assert(i > 0);
if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) { if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
// an optional rule that failed, so continue with next arg // an optional rule that failed, so continue with next arg
push_result_node(parser, MP_PARSE_NODE_NULL); push_result_node(&parser, MP_PARSE_NODE_NULL);
backtrack = false; backtrack = false;
} else { } else {
// a mandatory rule that failed, so propagate backtrack // a mandatory rule that failed, so propagate backtrack
@ -463,7 +468,7 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
if (mp_lexer_is_kind(lex, tok_kind)) { if (mp_lexer_is_kind(lex, tok_kind)) {
// matched token // matched token
if (tok_kind == MP_TOKEN_NAME) { if (tok_kind == MP_TOKEN_NAME) {
push_result_token(parser, lex); push_result_token(&parser, lex);
} }
mp_lexer_to_next(lex); mp_lexer_to_next(lex);
} else { } else {
@ -480,8 +485,8 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
break; break;
case RULE_ARG_RULE: case RULE_ARG_RULE:
case RULE_ARG_OPT_RULE: case RULE_ARG_OPT_RULE:
push_rule(parser, rule_src_line, rule, i + 1); // save this and-rule push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule
push_rule_from_arg(parser, rule->arg[i]); // push child of and-rule push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule
goto next_rule; goto next_rule;
default: default:
assert(0); assert(0);
@ -514,12 +519,12 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
#if 0 && !MICROPY_ENABLE_DOC_STRING #if 0 && !MICROPY_ENABLE_DOC_STRING
// this code discards lonely statement, such as doc strings // this code discards lonely statement, such as doc strings
// problem is that doc strings have already been interned, so this doesn't really help reduce RAM usage // problem is that doc strings have already been interned, so this doesn't really help reduce RAM usage
if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(parser, 0) == MP_PARSE_NODE_NULL) { if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) {
mp_parse_node_t p = peek_result(parser, 1); mp_parse_node_t p = peek_result(&parser, 1);
if (MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) { if (MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) {
pop_result(parser); pop_result(parser);
pop_result(parser); pop_result(parser);
push_result_rule(parser, rule_src_line, rules[RULE_pass_stmt], 0); push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0);
break; break;
} }
} }
@ -540,35 +545,35 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
// always emit these rules, and add an extra blank node at the end (to be used by the compiler to store data) // always emit these rules, and add an extra blank node at the end (to be used by the compiler to store data)
if (ADD_BLANK_NODE(rule->rule_id)) { if (ADD_BLANK_NODE(rule->rule_id)) {
emit_rule = true; emit_rule = true;
push_result_node(parser, MP_PARSE_NODE_NULL); push_result_node(&parser, MP_PARSE_NODE_NULL);
i += 1; i += 1;
} }
int num_not_nil = 0; int num_not_nil = 0;
for (int x = 0; x < i; ++x) { for (int x = 0; x < i; ++x) {
if (peek_result(parser, x) != MP_PARSE_NODE_NULL) { if (peek_result(&parser, x) != MP_PARSE_NODE_NULL) {
num_not_nil += 1; num_not_nil += 1;
} }
} }
//printf("done and %s n=%d i=%d notnil=%d\n", rule->rule_name, n, i, num_not_nil); //printf("done and %s n=%d i=%d notnil=%d\n", rule->rule_name, n, i, num_not_nil);
if (emit_rule) { if (emit_rule) {
push_result_rule(parser, rule_src_line, rule, i); push_result_rule(&parser, rule_src_line, rule, i);
} else if (num_not_nil == 0) { } else if (num_not_nil == 0) {
push_result_rule(parser, rule_src_line, rule, i); // needed for, eg, atom_paren, testlist_comp_3b push_result_rule(&parser, rule_src_line, rule, i); // needed for, eg, atom_paren, testlist_comp_3b
//result_stack_show(parser); //result_stack_show(parser);
//assert(0); //assert(0);
} else if (num_not_nil == 1) { } else if (num_not_nil == 1) {
// single result, leave it on stack // single result, leave it on stack
mp_parse_node_t pn = MP_PARSE_NODE_NULL; mp_parse_node_t pn = MP_PARSE_NODE_NULL;
for (int x = 0; x < i; ++x) { for (int x = 0; x < i; ++x) {
mp_parse_node_t pn2 = pop_result(parser); mp_parse_node_t pn2 = pop_result(&parser);
if (pn2 != MP_PARSE_NODE_NULL) { if (pn2 != MP_PARSE_NODE_NULL) {
pn = pn2; pn = pn2;
} }
} }
push_result_node(parser, pn); push_result_node(&parser, pn);
} else { } else {
push_result_rule(parser, rule_src_line, rule, i); push_result_rule(&parser, rule_src_line, rule, i);
} }
break; break;
@ -615,7 +620,7 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
if (i & 1 & n) { if (i & 1 & n) {
// separators which are tokens are not pushed to result stack // separators which are tokens are not pushed to result stack
} else { } else {
push_result_token(parser, lex); push_result_token(&parser, lex);
} }
mp_lexer_to_next(lex); mp_lexer_to_next(lex);
// got element of list, so continue parsing list // got element of list, so continue parsing list
@ -628,8 +633,8 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
} }
break; break;
case RULE_ARG_RULE: case RULE_ARG_RULE:
push_rule(parser, rule_src_line, rule, i + 1); // save this list-rule push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule
push_rule_from_arg(parser, arg); // push child of list-rule push_rule_from_arg(&parser, arg); // push child of list-rule
goto next_rule; goto next_rule;
default: default:
assert(0); assert(0);
@ -649,13 +654,13 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
// list matched single item // list matched single item
if (had_trailing_sep) { if (had_trailing_sep) {
// if there was a trailing separator, make a list of a single item // if there was a trailing separator, make a list of a single item
push_result_rule(parser, rule_src_line, rule, i); push_result_rule(&parser, rule_src_line, rule, i);
} else { } else {
// just leave single item on stack (ie don't wrap in a list) // just leave single item on stack (ie don't wrap in a list)
} }
} else { } else {
//printf("done list %s %d %d\n", rule->rule_name, n, i); //printf("done list %s %d %d\n", rule->rule_name, n, i);
push_result_rule(parser, rule_src_line, rule, i); push_result_rule(&parser, rule_src_line, rule, i);
} }
break; break;
@ -667,7 +672,8 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
mp_parse_node_t result; mp_parse_node_t result;
// check if we had a memory error // check if we had a memory error
if (parser->had_memory_error) { if (parser.had_memory_error) {
memory_error:
*parse_error_kind_out = MP_PARSE_ERROR_MEMORY; *parse_error_kind_out = MP_PARSE_ERROR_MEMORY;
result = MP_PARSE_NODE_NULL; result = MP_PARSE_NODE_NULL;
goto finished; goto finished;
@ -681,19 +687,18 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
//printf("--------------\n"); //printf("--------------\n");
//result_stack_show(parser); //result_stack_show(parser);
//printf("rule stack alloc: %d\n", parser->rule_stack_alloc); //printf("rule stack alloc: %d\n", parser.rule_stack_alloc);
//printf("result stack alloc: %d\n", parser->result_stack_alloc); //printf("result stack alloc: %d\n", parser.result_stack_alloc);
//printf("number of parse nodes allocated: %d\n", num_parse_nodes_allocated); //printf("number of parse nodes allocated: %d\n", num_parse_nodes_allocated);
// get the root parse node that we created // get the root parse node that we created
assert(parser->result_stack_top == 1); assert(parser.result_stack_top == 1);
result = parser->result_stack[0]; result = parser.result_stack[0];
finished: finished:
// free the memory that we don't need anymore // free the memory that we don't need anymore
m_del(rule_stack_t, parser->rule_stack, parser->rule_stack_alloc); m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc);
m_del(mp_parse_node_t, parser->result_stack, parser->result_stack_alloc); m_del(mp_parse_node_t, parser.result_stack, parser.result_stack_alloc);
m_del_obj(parser_t, parser);
// return the result // return the result
return result; return result;