djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

py/lexer: Simplify handling of indenting of very first token.

Browse Source
This commit is contained in:
Damien George 2017-02-17 10:56:06 +11:00
parent b0599de48e
commit 98b3072da5
1 changed files with 10 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
py/lexer.c
View File

@ -283,7 +283,7 @@ STATIC bool get_hex(mp_lexer_t *lex, mp_uint_t num_digits, mp_uint_t *result) {
return true;
}
STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) {
void mp_lexer_to_next(mp_lexer_t *lex) {
// start new token text
vstr_reset(&lex->vstr);
@ -322,14 +322,7 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) {
lex->tok_line = lex->line;
lex->tok_column = lex->column;
if (first_token && lex->line == 1 && lex->column != 1) {
// check that the first token is in the first column
// if first token is not on first line, we get a physical newline and
// this check is done as part of normal indent/dedent checking below
// (done to get equivalence with CPython)
lex->tok_kind = MP_TOKEN_INDENT;
} else if (lex->emit_dent < 0) {
if (lex->emit_dent < 0) {
lex->tok_kind = MP_TOKEN_DEDENT;
lex->emit_dent += 1;
@ -705,7 +698,7 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) {
vstr_init(&lex->vstr, 32);
// check for memory allocation error
// note: vstr_init above may fail on malloc, but so may mp_lexer_next_token_into below
// note: vstr_init above may fail on malloc, but so may mp_lexer_to_next below
if (lex->indent_level == NULL) {
mp_lexer_free(lex);
return NULL;
@ -737,7 +730,13 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) {
}
// preload first token
mp_lexer_next_token_into(lex, true);
mp_lexer_to_next(lex);
// Check that the first token is in the first column. If it's not then we
// convert the token kind to INDENT so that the parser gives a syntax error.
if (lex->tok_column != 1) {
lex->tok_kind = MP_TOKEN_INDENT;
}
return lex;
}
@ -785,10 +784,6 @@ void mp_lexer_free(mp_lexer_t *lex) {
}
}
void mp_lexer_to_next(mp_lexer_t *lex) {
mp_lexer_next_token_into(lex, false);
}
#if 0
// This function is used to print the current token and should only be
// needed to debug the lexer, so it's not available via a config option.