djsundog
/
circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Integrate new lexer stream with stm framework.

This commit is contained in:
Damien 2013-10-20 17:42:00 +01:00
parent 27fb45eb1c
commit fa2162bc77
10 changed files with 126 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
py/lexer.c
View File

@ -14,7 +14,7 @@ struct _py_lexer_t {
const char *name; // name of source
void *stream_data; // data for stream
py_lexer_stream_next_char_t stream_next_char; // stream callback to get next char
py_lexer_stream_free_t stream_free; // stream callback to free
py_lexer_stream_close_t stream_close; // stream callback to free
unichar chr0, chr1, chr2; // current cached characters from source
@ -589,13 +589,13 @@ static void py_lexer_next_token_into(py_lexer_t *lex, py_token_t *tok, bool firs
}
}
py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_free_t stream_free) {
py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_close_t stream_close) {
py_lexer_t *lex = m_new(py_lexer_t, 1);
lex->name = src_name; // TODO do we need to strdup this?
lex->stream_data = stream_data;
lex->stream_next_char = stream_next_char;
lex->stream_free = stream_free;
lex->stream_close = stream_close;
lex->line = 1;
lex->column = 1;
lex->emit_dent = 0;
@ -632,8 +632,8 @@ py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_strea
void py_lexer_free(py_lexer_t *lex) {
if (lex) {
if (lex->stream_free) {
lex->stream_free(lex->stream_data);
if (lex->stream_close) {
lex->stream_close(lex->stream_data);
}
m_free(lex);
}

4
py/lexer.h
View File

@ -122,7 +122,7 @@ typedef struct _py_token_t {
// it can be called again after returning PY_LEXER_CHAR_EOF, and in that case must return PY_LEXER_CHAR_EOF
#define PY_LEXER_CHAR_EOF (-1)
typedef unichar (*py_lexer_stream_next_char_t)(void*);
typedef void (*py_lexer_stream_free_t)(void*);
typedef void (*py_lexer_stream_close_t)(void*);
typedef struct _py_lexer_t py_lexer_t;
@ -130,7 +130,7 @@ void py_token_show(const py_token_t *tok);
void py_token_show_error_prefix(const py_token_t *tok);
bool py_token_show_error(const py_token_t *tok, const char *msg);
py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_free_t stream_free);
py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_close_t stream_close);
void py_lexer_free(py_lexer_t *lex);
void py_lexer_to_next(py_lexer_t *lex);
const py_token_t *py_lexer_cur(const py_lexer_t *lex);

2
stm/Makefile
View File

@ -24,6 +24,7 @@ SRC_C = \
systick.c \
stm32fxxx_it.c \
usb.c \
lexerstm.c \
# sd.c \
SRC_S = \
@ -33,6 +34,7 @@ PY_O = \
nlrthumb.o \
malloc.o \
qstr.o \
vstr.o \
misc.o \
lexer.o \
parse.o \

60
stm/lexerstm.c Normal file
View File

@ -0,0 +1,60 @@
#include <stdint.h>
#include <stdio.h>
#include "ff.h"
#include "misc.h"
#include "lexer.h"
#include "lexerstm.h"
unichar str_buf_next_char(py_lexer_str_buf_t *sb) {
if (sb->src_cur < sb->src_end) {
return *sb->src_cur++;
} else {
return PY_LEXER_CHAR_EOF;
}
}
void str_buf_free(py_lexer_str_buf_t *sb) {
if (sb->free) {
m_free((char*)sb->src_beg);
}
}
py_lexer_t *py_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, py_lexer_str_buf_t *sb) {
sb->free = free_str;
sb->src_beg = str;
sb->src_cur = str;
sb->src_end = str + len;
return py_lexer_new(src_name, sb, (py_lexer_stream_next_char_t)str_buf_next_char, (py_lexer_stream_close_t)str_buf_free);
}
unichar file_buf_next_char(py_lexer_file_buf_t *fb) {
if (fb->pos >= fb->len) {
if (fb->len < sizeof(fb->buf)) {
return PY_LEXER_CHAR_EOF;
} else {
UINT n;
f_read(&fb->fp, fb->buf, sizeof(fb->buf), &n);
fb->len = n;
fb->pos = 0;
}
}
return fb->buf[fb->pos++];
}
void file_buf_close(py_lexer_file_buf_t *fb) {
f_close(&fb->fp);
}
py_lexer_t *py_lexer_new_from_file(const char *filename, py_lexer_file_buf_t *fb) {
FRESULT res = f_open(&fb->fp, filename, FA_READ);
if (res != FR_OK) {
return NULL;
}
UINT n;
f_read(&fb->fp, fb->buf, sizeof(fb->buf), &n);
fb->len = n;
fb->pos = 0;
return py_lexer_new(filename, fb, (py_lexer_stream_next_char_t)file_buf_next_char, (py_lexer_stream_close_t)file_buf_close);
}

16
stm/lexerstm.h Normal file
View File

@ -0,0 +1,16 @@
typedef struct _py_lexer_str_buf_t {
bool free; // free src_beg when done
const char *src_beg; // beginning of source
const char *src_cur; // current location in source
const char *src_end; // end (exclusive) of source
} py_lexer_str_buf_t;
typedef struct _py_lexer_file_buf_t {
FIL fp;
char buf[20];
uint16_t len;
uint16_t pos;
} py_lexer_file_buf_t;
py_lexer_t *py_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, py_lexer_str_buf_t *sb);
py_lexer_t *py_lexer_new_from_file(const char *filename, py_lexer_file_buf_t *fb);

40
stm/main.c
View File

@ -11,6 +11,7 @@
#include "storage.h"
#include "mma.h"
#include "usb.h"
#include "ff.h"
static void impl02_c_version() {
int x = 0;
@ -63,8 +64,10 @@ void __fatal_error(const char *msg) {
}
}
#include "nlr.h"
#include "misc.h"
#include "lexer.h"
#include "lexerstm.h"
#include "mpyconfig.h"
#include "parse.h"
#include "compile.h"
@ -88,10 +91,8 @@ py_obj_t pyb_sw() {
}
}
#include "ff.h"
FATFS fatfs0;
#include "nlr.h"
/*
void g(uint i) {
@ -293,7 +294,7 @@ int main() {
//sys_tick_delay_ms(1000);
// Python!
if (0) {
if (1) {
//const char *pysrc = "def f():\n x=x+1\nprint(42)\n";
const char *pysrc =
// impl01.py
@ -323,7 +324,6 @@ int main() {
" x = x + 1\n"
"f()\n";
*/
/*
"print('in python!')\n"
"x = 0\n"
"while x < 4:\n"
@ -331,11 +331,10 @@ int main() {
" pyb_delay(201)\n"
" pyb_led(False)\n"
" pyb_delay(201)\n"
" x = x + 1\n"
" x += 1\n"
"print('press me!')\n"
"while True:\n"
" pyb_led(pyb_sw())\n";
*/
/*
// impl16.py
"@micropython.asm_thumb\n"
@ -372,6 +371,7 @@ int main() {
"except:\n"
" print(x)\n";
*/
/*
// impl19.py
"# for loop\n"
"def f():\n"
@ -380,29 +380,27 @@ int main() {
" for z in range(400):\n"
" pass\n"
"f()\n";
*/
py_lexer_t *lex = py_lexer_from_str_len("<>", pysrc, strlen(pysrc), false);
py_lexer_str_buf_t py_lexer_str_buf;
py_lexer_t *lex = py_lexer_new_from_str_len("<stdin>", pysrc, strlen(pysrc), false, &py_lexer_str_buf);
if (0) {
while (!py_lexer_is_kind(lex, PY_TOKEN_END)) {
py_token_show(py_lexer_cur(lex));
py_lexer_to_next(lex);
sys_tick_delay_ms(1000);
}
} else {
// nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler
printf("lex; al=%u\n", m_get_total_bytes_allocated());
sys_tick_delay_ms(1000);
py_parse_node_t pn = py_parse(lex, 0);
//printf("----------------\n");
// nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler
printf("lex; al=%u\n", m_get_total_bytes_allocated());
sys_tick_delay_ms(1000);
py_parse_node_t pn = py_parse(lex, PY_PARSE_FILE_INPUT);
py_lexer_free(lex);
if (pn != PY_PARSE_NODE_NULL) {
printf("pars;al=%u\n", m_get_total_bytes_allocated());
sys_tick_delay_ms(1000);
//parse_node_show(pn, 0);
py_compile(pn, false);
bool comp_ok = py_compile(pn, false);
printf("comp;al=%u\n", m_get_total_bytes_allocated());
sys_tick_delay_ms(1000);
if (1) {
if (!comp_ok) {
printf("compile error\n");
} else {
// execute it!
// add some functions to the python namespace

1
stm/std.h
View File

@ -8,6 +8,7 @@ void *calloc(size_t sz, size_t n);
void *realloc(void *ptr, size_t n);
void *memcpy(void *dest, const void *src, size_t n);
void *memmove(void *dest, const void *src, size_t n);
void *memset(void *s, int c, size_t n);
int strlen(const char *str);

15
stm/string0.c
View File

@ -11,6 +11,21 @@ void *memcpy(void *dest, const void *src, size_t n) {
return dest;
}
void *memmove(void *dest, const void *src, size_t n) {
if (src < dest && dest < src + n) {
// need to copy backwards
uint8_t *d = dest + n - 1;
const uint8_t *s = src + n - 1;
for (; n > 0; n--) {
*d-- = *s--;
}
return dest;
} else {
// can use normal memcpy
return memcpy(dest, src, n);
}
}
void *memset(void *s, int c, size_t n) {
uint8_t *s2 = s;
for (; n > 0; n--) {

7
unix/main.c
View File

@ -68,8 +68,11 @@ void do_repl() {
line = line3;
}
}
py_lexer_t *lex = py_lexer_new_from_str_len("<stdin>", line, strlen(line), false);
py_parse_node_t pn = py_parse(lex, PY_PARSE_SINGLE_INPUT);
py_lexer_free(lex);
if (pn != PY_PARSE_NODE_NULL) {
//py_parse_node_show(pn, 0);
bool comp_ok = py_compile(pn, true);
@ -111,6 +114,8 @@ void do_file(const char *file) {
// compile
py_parse_node_t pn = py_parse(lex, PY_PARSE_FILE_INPUT);
py_lexer_free(lex);
if (pn != PY_PARSE_NODE_NULL) {
//printf("----------------\n");
//parse_node_show(pn, 0);
@ -118,8 +123,6 @@ void do_file(const char *file) {
bool comp_ok = py_compile(pn, false);
//printf("----------------\n");
py_lexer_free(lex);
#if MICROPY_EMIT_CPYTHON
if (!comp_ok) {
printf("compile error\n");

2
unix/mpyconfig.h
View File

@ -1,7 +1,7 @@
// options to control how Micro Python is built
#define MICROPY_ENABLE_FLOAT (1)
#define MICROPY_EMIT_CPYTHON (1)
#define MICROPY_EMIT_CPYTHON (0)
#define MICROPY_EMIT_X64 (1)
#define MICROPY_EMIT_THUMB (0)
#define MICROPY_EMIT_INLINE_THUMB (0)